1/*===- TableGen'erated file -------------------------------------*- C++ -*-===*\
2|* *|
3|* "Fast" Instruction Selector for the AArch64 target *|
4|* *|
5|* Automatically generated file, do not edit! *|
6|* *|
7\*===----------------------------------------------------------------------===*/
8
9
10// FastEmit Immediate Predicate functions.
11static bool Predicate_imm0_31(int64_t Imm) {
12
13 return ((uint64_t)Imm) < 32;
14
15}
16static bool Predicate_imm0_63(int64_t Imm) {
17
18 return ((uint64_t)Imm) < 64;
19
20}
21static bool Predicate_imm32_0_31(int64_t Imm) {
22
23 return ((uint64_t)Imm) < 32;
24
25}
26static bool Predicate_tbz_imm0_31_diag(int64_t Imm) {
27
28 return (((uint32_t)Imm) < 32);
29
30}
31static bool Predicate_tbz_imm32_63(int64_t Imm) {
32
33 return (((uint32_t)Imm) > 31) && (((uint32_t)Imm) < 64);
34
35}
36static bool Predicate_VectorIndexD(int64_t Imm) {
37 return ((uint64_t)Imm) < 2;
38}
39static bool Predicate_VectorIndexS(int64_t Imm) {
40 return ((uint64_t)Imm) < 4;
41}
42static bool Predicate_VectorIndexH(int64_t Imm) {
43 return ((uint64_t)Imm) < 8;
44}
45static bool Predicate_VectorIndexB(int64_t Imm) {
46 return ((uint64_t)Imm) < 16;
47}
48static bool Predicate_imm0_255(int64_t Imm) {
49
50 return ((uint32_t)Imm) < 256;
51
52}
53static bool Predicate_vecshiftL64(int64_t Imm) {
54
55 return (((uint32_t)Imm) < 64);
56
57}
58static bool Predicate_vecshiftL32(int64_t Imm) {
59
60 return (((uint32_t)Imm) < 32);
61
62}
63static bool Predicate_vecshiftR64(int64_t Imm) {
64
65 return (((uint32_t)Imm) > 0) && (((uint32_t)Imm) < 65);
66
67}
68static bool Predicate_vecshiftL8(int64_t Imm) {
69
70 return (((uint32_t)Imm) < 8);
71
72}
73static bool Predicate_vecshiftL16(int64_t Imm) {
74
75 return (((uint32_t)Imm) < 16);
76
77}
78static bool Predicate_vecshiftR8(int64_t Imm) {
79
80 return (((uint32_t)Imm) > 0) && (((uint32_t)Imm) < 9);
81
82}
83static bool Predicate_vecshiftR16(int64_t Imm) {
84
85 return (((uint32_t)Imm) > 0) && (((uint32_t)Imm) < 17);
86
87}
88static bool Predicate_vecshiftR32(int64_t Imm) {
89
90 return (((uint32_t)Imm) > 0) && (((uint32_t)Imm) < 33);
91
92}
93static bool Predicate_simm5_8b(int64_t Imm) {
94 return (int8_t)Imm >= -16 && (int8_t)Imm < 16;
95}
96static bool Predicate_simm5_16b(int64_t Imm) {
97 return (int16_t)Imm >= -16 && (int16_t)Imm < 16;
98}
99static bool Predicate_simm5_32b(int64_t Imm) {
100 return Imm >= -16 && Imm < 16;
101}
102static bool Predicate_simm5_64b(int64_t Imm) {
103 return Imm >= -16 && Imm < 16;
104}
105
106
107// FastEmit functions for AArch64ISD::THREAD_POINTER.
108
109unsigned fastEmit_AArch64ISD_THREAD_POINTER_MVT_i64_(MVT RetVT) {
110 if (RetVT.SimpleTy != MVT::i64)
111 return 0;
112 return fastEmitInst_(AArch64::MOVbaseTLS, &AArch64::GPR64RegClass);
113}
114
115unsigned fastEmit_AArch64ISD_THREAD_POINTER_(MVT VT, MVT RetVT) {
116 switch (VT.SimpleTy) {
117 case MVT::i64: return fastEmit_AArch64ISD_THREAD_POINTER_MVT_i64_(RetVT);
118 default: return 0;
119 }
120}
121
122// Top-level FastEmit function.
123
124unsigned fastEmit_(MVT VT, MVT RetVT, unsigned Opcode) override {
125 switch (Opcode) {
126 case AArch64ISD::THREAD_POINTER: return fastEmit_AArch64ISD_THREAD_POINTER_(VT, RetVT);
127 default: return 0;
128 }
129}
130
131// FastEmit functions for AArch64ISD::CALL.
132
133unsigned fastEmit_AArch64ISD_CALL_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
134 if (RetVT.SimpleTy != MVT::isVoid)
135 return 0;
136 if (( MF->getSubtarget<AArch64Subtarget>().hardenSlsBlr() )) {
137 return fastEmitInst_r(AArch64::BLRNoIP, &AArch64::GPR64noipRegClass, Op0, Op0IsKill);
138 }
139 if (( !MF->getSubtarget<AArch64Subtarget>().hardenSlsBlr() )) {
140 return fastEmitInst_r(AArch64::BLR, &AArch64::GPR64RegClass, Op0, Op0IsKill);
141 }
142 return 0;
143}
144
145unsigned fastEmit_AArch64ISD_CALL_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
146 switch (VT.SimpleTy) {
147 case MVT::i64: return fastEmit_AArch64ISD_CALL_MVT_i64_r(RetVT, Op0, Op0IsKill);
148 default: return 0;
149 }
150}
151
152// FastEmit functions for AArch64ISD::CMEQz.
153
154unsigned fastEmit_AArch64ISD_CMEQz_MVT_v8i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
155 if (RetVT.SimpleTy != MVT::v8i8)
156 return 0;
157 if ((Subtarget->hasNEON())) {
158 return fastEmitInst_r(AArch64::CMEQv8i8rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
159 }
160 return 0;
161}
162
163unsigned fastEmit_AArch64ISD_CMEQz_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
164 if (RetVT.SimpleTy != MVT::v16i8)
165 return 0;
166 if ((Subtarget->hasNEON())) {
167 return fastEmitInst_r(AArch64::CMEQv16i8rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
168 }
169 return 0;
170}
171
172unsigned fastEmit_AArch64ISD_CMEQz_MVT_v4i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
173 if (RetVT.SimpleTy != MVT::v4i16)
174 return 0;
175 if ((Subtarget->hasNEON())) {
176 return fastEmitInst_r(AArch64::CMEQv4i16rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
177 }
178 return 0;
179}
180
181unsigned fastEmit_AArch64ISD_CMEQz_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
182 if (RetVT.SimpleTy != MVT::v8i16)
183 return 0;
184 if ((Subtarget->hasNEON())) {
185 return fastEmitInst_r(AArch64::CMEQv8i16rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
186 }
187 return 0;
188}
189
190unsigned fastEmit_AArch64ISD_CMEQz_MVT_v2i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
191 if (RetVT.SimpleTy != MVT::v2i32)
192 return 0;
193 if ((Subtarget->hasNEON())) {
194 return fastEmitInst_r(AArch64::CMEQv2i32rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
195 }
196 return 0;
197}
198
199unsigned fastEmit_AArch64ISD_CMEQz_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
200 if (RetVT.SimpleTy != MVT::v4i32)
201 return 0;
202 if ((Subtarget->hasNEON())) {
203 return fastEmitInst_r(AArch64::CMEQv4i32rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
204 }
205 return 0;
206}
207
208unsigned fastEmit_AArch64ISD_CMEQz_MVT_v1i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
209 if (RetVT.SimpleTy != MVT::v1i64)
210 return 0;
211 if ((Subtarget->hasNEON())) {
212 return fastEmitInst_r(AArch64::CMEQv1i64rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
213 }
214 return 0;
215}
216
217unsigned fastEmit_AArch64ISD_CMEQz_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
218 if (RetVT.SimpleTy != MVT::v2i64)
219 return 0;
220 if ((Subtarget->hasNEON())) {
221 return fastEmitInst_r(AArch64::CMEQv2i64rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
222 }
223 return 0;
224}
225
226unsigned fastEmit_AArch64ISD_CMEQz_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
227 switch (VT.SimpleTy) {
228 case MVT::v8i8: return fastEmit_AArch64ISD_CMEQz_MVT_v8i8_r(RetVT, Op0, Op0IsKill);
229 case MVT::v16i8: return fastEmit_AArch64ISD_CMEQz_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
230 case MVT::v4i16: return fastEmit_AArch64ISD_CMEQz_MVT_v4i16_r(RetVT, Op0, Op0IsKill);
231 case MVT::v8i16: return fastEmit_AArch64ISD_CMEQz_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
232 case MVT::v2i32: return fastEmit_AArch64ISD_CMEQz_MVT_v2i32_r(RetVT, Op0, Op0IsKill);
233 case MVT::v4i32: return fastEmit_AArch64ISD_CMEQz_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
234 case MVT::v1i64: return fastEmit_AArch64ISD_CMEQz_MVT_v1i64_r(RetVT, Op0, Op0IsKill);
235 case MVT::v2i64: return fastEmit_AArch64ISD_CMEQz_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
236 default: return 0;
237 }
238}
239
240// FastEmit functions for AArch64ISD::CMGEz.
241
242unsigned fastEmit_AArch64ISD_CMGEz_MVT_v8i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
243 if (RetVT.SimpleTy != MVT::v8i8)
244 return 0;
245 if ((Subtarget->hasNEON())) {
246 return fastEmitInst_r(AArch64::CMGEv8i8rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
247 }
248 return 0;
249}
250
251unsigned fastEmit_AArch64ISD_CMGEz_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
252 if (RetVT.SimpleTy != MVT::v16i8)
253 return 0;
254 if ((Subtarget->hasNEON())) {
255 return fastEmitInst_r(AArch64::CMGEv16i8rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
256 }
257 return 0;
258}
259
260unsigned fastEmit_AArch64ISD_CMGEz_MVT_v4i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
261 if (RetVT.SimpleTy != MVT::v4i16)
262 return 0;
263 if ((Subtarget->hasNEON())) {
264 return fastEmitInst_r(AArch64::CMGEv4i16rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
265 }
266 return 0;
267}
268
269unsigned fastEmit_AArch64ISD_CMGEz_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
270 if (RetVT.SimpleTy != MVT::v8i16)
271 return 0;
272 if ((Subtarget->hasNEON())) {
273 return fastEmitInst_r(AArch64::CMGEv8i16rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
274 }
275 return 0;
276}
277
278unsigned fastEmit_AArch64ISD_CMGEz_MVT_v2i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
279 if (RetVT.SimpleTy != MVT::v2i32)
280 return 0;
281 if ((Subtarget->hasNEON())) {
282 return fastEmitInst_r(AArch64::CMGEv2i32rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
283 }
284 return 0;
285}
286
287unsigned fastEmit_AArch64ISD_CMGEz_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
288 if (RetVT.SimpleTy != MVT::v4i32)
289 return 0;
290 if ((Subtarget->hasNEON())) {
291 return fastEmitInst_r(AArch64::CMGEv4i32rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
292 }
293 return 0;
294}
295
296unsigned fastEmit_AArch64ISD_CMGEz_MVT_v1i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
297 if (RetVT.SimpleTy != MVT::v1i64)
298 return 0;
299 if ((Subtarget->hasNEON())) {
300 return fastEmitInst_r(AArch64::CMGEv1i64rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
301 }
302 return 0;
303}
304
305unsigned fastEmit_AArch64ISD_CMGEz_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
306 if (RetVT.SimpleTy != MVT::v2i64)
307 return 0;
308 if ((Subtarget->hasNEON())) {
309 return fastEmitInst_r(AArch64::CMGEv2i64rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
310 }
311 return 0;
312}
313
314unsigned fastEmit_AArch64ISD_CMGEz_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
315 switch (VT.SimpleTy) {
316 case MVT::v8i8: return fastEmit_AArch64ISD_CMGEz_MVT_v8i8_r(RetVT, Op0, Op0IsKill);
317 case MVT::v16i8: return fastEmit_AArch64ISD_CMGEz_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
318 case MVT::v4i16: return fastEmit_AArch64ISD_CMGEz_MVT_v4i16_r(RetVT, Op0, Op0IsKill);
319 case MVT::v8i16: return fastEmit_AArch64ISD_CMGEz_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
320 case MVT::v2i32: return fastEmit_AArch64ISD_CMGEz_MVT_v2i32_r(RetVT, Op0, Op0IsKill);
321 case MVT::v4i32: return fastEmit_AArch64ISD_CMGEz_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
322 case MVT::v1i64: return fastEmit_AArch64ISD_CMGEz_MVT_v1i64_r(RetVT, Op0, Op0IsKill);
323 case MVT::v2i64: return fastEmit_AArch64ISD_CMGEz_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
324 default: return 0;
325 }
326}
327
328// FastEmit functions for AArch64ISD::CMGTz.
329
330unsigned fastEmit_AArch64ISD_CMGTz_MVT_v8i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
331 if (RetVT.SimpleTy != MVT::v8i8)
332 return 0;
333 if ((Subtarget->hasNEON())) {
334 return fastEmitInst_r(AArch64::CMGTv8i8rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
335 }
336 return 0;
337}
338
339unsigned fastEmit_AArch64ISD_CMGTz_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
340 if (RetVT.SimpleTy != MVT::v16i8)
341 return 0;
342 if ((Subtarget->hasNEON())) {
343 return fastEmitInst_r(AArch64::CMGTv16i8rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
344 }
345 return 0;
346}
347
348unsigned fastEmit_AArch64ISD_CMGTz_MVT_v4i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
349 if (RetVT.SimpleTy != MVT::v4i16)
350 return 0;
351 if ((Subtarget->hasNEON())) {
352 return fastEmitInst_r(AArch64::CMGTv4i16rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
353 }
354 return 0;
355}
356
357unsigned fastEmit_AArch64ISD_CMGTz_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
358 if (RetVT.SimpleTy != MVT::v8i16)
359 return 0;
360 if ((Subtarget->hasNEON())) {
361 return fastEmitInst_r(AArch64::CMGTv8i16rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
362 }
363 return 0;
364}
365
366unsigned fastEmit_AArch64ISD_CMGTz_MVT_v2i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
367 if (RetVT.SimpleTy != MVT::v2i32)
368 return 0;
369 if ((Subtarget->hasNEON())) {
370 return fastEmitInst_r(AArch64::CMGTv2i32rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
371 }
372 return 0;
373}
374
375unsigned fastEmit_AArch64ISD_CMGTz_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
376 if (RetVT.SimpleTy != MVT::v4i32)
377 return 0;
378 if ((Subtarget->hasNEON())) {
379 return fastEmitInst_r(AArch64::CMGTv4i32rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
380 }
381 return 0;
382}
383
384unsigned fastEmit_AArch64ISD_CMGTz_MVT_v1i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
385 if (RetVT.SimpleTy != MVT::v1i64)
386 return 0;
387 if ((Subtarget->hasNEON())) {
388 return fastEmitInst_r(AArch64::CMGTv1i64rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
389 }
390 return 0;
391}
392
393unsigned fastEmit_AArch64ISD_CMGTz_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
394 if (RetVT.SimpleTy != MVT::v2i64)
395 return 0;
396 if ((Subtarget->hasNEON())) {
397 return fastEmitInst_r(AArch64::CMGTv2i64rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
398 }
399 return 0;
400}
401
402unsigned fastEmit_AArch64ISD_CMGTz_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
403 switch (VT.SimpleTy) {
404 case MVT::v8i8: return fastEmit_AArch64ISD_CMGTz_MVT_v8i8_r(RetVT, Op0, Op0IsKill);
405 case MVT::v16i8: return fastEmit_AArch64ISD_CMGTz_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
406 case MVT::v4i16: return fastEmit_AArch64ISD_CMGTz_MVT_v4i16_r(RetVT, Op0, Op0IsKill);
407 case MVT::v8i16: return fastEmit_AArch64ISD_CMGTz_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
408 case MVT::v2i32: return fastEmit_AArch64ISD_CMGTz_MVT_v2i32_r(RetVT, Op0, Op0IsKill);
409 case MVT::v4i32: return fastEmit_AArch64ISD_CMGTz_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
410 case MVT::v1i64: return fastEmit_AArch64ISD_CMGTz_MVT_v1i64_r(RetVT, Op0, Op0IsKill);
411 case MVT::v2i64: return fastEmit_AArch64ISD_CMGTz_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
412 default: return 0;
413 }
414}
415
416// FastEmit functions for AArch64ISD::CMLEz.
417
418unsigned fastEmit_AArch64ISD_CMLEz_MVT_v8i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
419 if (RetVT.SimpleTy != MVT::v8i8)
420 return 0;
421 if ((Subtarget->hasNEON())) {
422 return fastEmitInst_r(AArch64::CMLEv8i8rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
423 }
424 return 0;
425}
426
427unsigned fastEmit_AArch64ISD_CMLEz_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
428 if (RetVT.SimpleTy != MVT::v16i8)
429 return 0;
430 if ((Subtarget->hasNEON())) {
431 return fastEmitInst_r(AArch64::CMLEv16i8rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
432 }
433 return 0;
434}
435
436unsigned fastEmit_AArch64ISD_CMLEz_MVT_v4i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
437 if (RetVT.SimpleTy != MVT::v4i16)
438 return 0;
439 if ((Subtarget->hasNEON())) {
440 return fastEmitInst_r(AArch64::CMLEv4i16rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
441 }
442 return 0;
443}
444
445unsigned fastEmit_AArch64ISD_CMLEz_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
446 if (RetVT.SimpleTy != MVT::v8i16)
447 return 0;
448 if ((Subtarget->hasNEON())) {
449 return fastEmitInst_r(AArch64::CMLEv8i16rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
450 }
451 return 0;
452}
453
454unsigned fastEmit_AArch64ISD_CMLEz_MVT_v2i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
455 if (RetVT.SimpleTy != MVT::v2i32)
456 return 0;
457 if ((Subtarget->hasNEON())) {
458 return fastEmitInst_r(AArch64::CMLEv2i32rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
459 }
460 return 0;
461}
462
463unsigned fastEmit_AArch64ISD_CMLEz_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
464 if (RetVT.SimpleTy != MVT::v4i32)
465 return 0;
466 if ((Subtarget->hasNEON())) {
467 return fastEmitInst_r(AArch64::CMLEv4i32rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
468 }
469 return 0;
470}
471
472unsigned fastEmit_AArch64ISD_CMLEz_MVT_v1i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
473 if (RetVT.SimpleTy != MVT::v1i64)
474 return 0;
475 if ((Subtarget->hasNEON())) {
476 return fastEmitInst_r(AArch64::CMLEv1i64rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
477 }
478 return 0;
479}
480
481unsigned fastEmit_AArch64ISD_CMLEz_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
482 if (RetVT.SimpleTy != MVT::v2i64)
483 return 0;
484 if ((Subtarget->hasNEON())) {
485 return fastEmitInst_r(AArch64::CMLEv2i64rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
486 }
487 return 0;
488}
489
490unsigned fastEmit_AArch64ISD_CMLEz_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
491 switch (VT.SimpleTy) {
492 case MVT::v8i8: return fastEmit_AArch64ISD_CMLEz_MVT_v8i8_r(RetVT, Op0, Op0IsKill);
493 case MVT::v16i8: return fastEmit_AArch64ISD_CMLEz_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
494 case MVT::v4i16: return fastEmit_AArch64ISD_CMLEz_MVT_v4i16_r(RetVT, Op0, Op0IsKill);
495 case MVT::v8i16: return fastEmit_AArch64ISD_CMLEz_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
496 case MVT::v2i32: return fastEmit_AArch64ISD_CMLEz_MVT_v2i32_r(RetVT, Op0, Op0IsKill);
497 case MVT::v4i32: return fastEmit_AArch64ISD_CMLEz_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
498 case MVT::v1i64: return fastEmit_AArch64ISD_CMLEz_MVT_v1i64_r(RetVT, Op0, Op0IsKill);
499 case MVT::v2i64: return fastEmit_AArch64ISD_CMLEz_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
500 default: return 0;
501 }
502}
503
504// FastEmit functions for AArch64ISD::CMLTz.
505
506unsigned fastEmit_AArch64ISD_CMLTz_MVT_v8i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
507 if (RetVT.SimpleTy != MVT::v8i8)
508 return 0;
509 if ((Subtarget->hasNEON())) {
510 return fastEmitInst_r(AArch64::CMLTv8i8rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
511 }
512 return 0;
513}
514
515unsigned fastEmit_AArch64ISD_CMLTz_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
516 if (RetVT.SimpleTy != MVT::v16i8)
517 return 0;
518 if ((Subtarget->hasNEON())) {
519 return fastEmitInst_r(AArch64::CMLTv16i8rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
520 }
521 return 0;
522}
523
524unsigned fastEmit_AArch64ISD_CMLTz_MVT_v4i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
525 if (RetVT.SimpleTy != MVT::v4i16)
526 return 0;
527 if ((Subtarget->hasNEON())) {
528 return fastEmitInst_r(AArch64::CMLTv4i16rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
529 }
530 return 0;
531}
532
533unsigned fastEmit_AArch64ISD_CMLTz_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
534 if (RetVT.SimpleTy != MVT::v8i16)
535 return 0;
536 if ((Subtarget->hasNEON())) {
537 return fastEmitInst_r(AArch64::CMLTv8i16rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
538 }
539 return 0;
540}
541
542unsigned fastEmit_AArch64ISD_CMLTz_MVT_v2i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
543 if (RetVT.SimpleTy != MVT::v2i32)
544 return 0;
545 if ((Subtarget->hasNEON())) {
546 return fastEmitInst_r(AArch64::CMLTv2i32rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
547 }
548 return 0;
549}
550
551unsigned fastEmit_AArch64ISD_CMLTz_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
552 if (RetVT.SimpleTy != MVT::v4i32)
553 return 0;
554 if ((Subtarget->hasNEON())) {
555 return fastEmitInst_r(AArch64::CMLTv4i32rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
556 }
557 return 0;
558}
559
560unsigned fastEmit_AArch64ISD_CMLTz_MVT_v1i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
561 if (RetVT.SimpleTy != MVT::v1i64)
562 return 0;
563 if ((Subtarget->hasNEON())) {
564 return fastEmitInst_r(AArch64::CMLTv1i64rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
565 }
566 return 0;
567}
568
569unsigned fastEmit_AArch64ISD_CMLTz_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
570 if (RetVT.SimpleTy != MVT::v2i64)
571 return 0;
572 if ((Subtarget->hasNEON())) {
573 return fastEmitInst_r(AArch64::CMLTv2i64rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
574 }
575 return 0;
576}
577
578unsigned fastEmit_AArch64ISD_CMLTz_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
579 switch (VT.SimpleTy) {
580 case MVT::v8i8: return fastEmit_AArch64ISD_CMLTz_MVT_v8i8_r(RetVT, Op0, Op0IsKill);
581 case MVT::v16i8: return fastEmit_AArch64ISD_CMLTz_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
582 case MVT::v4i16: return fastEmit_AArch64ISD_CMLTz_MVT_v4i16_r(RetVT, Op0, Op0IsKill);
583 case MVT::v8i16: return fastEmit_AArch64ISD_CMLTz_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
584 case MVT::v2i32: return fastEmit_AArch64ISD_CMLTz_MVT_v2i32_r(RetVT, Op0, Op0IsKill);
585 case MVT::v4i32: return fastEmit_AArch64ISD_CMLTz_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
586 case MVT::v1i64: return fastEmit_AArch64ISD_CMLTz_MVT_v1i64_r(RetVT, Op0, Op0IsKill);
587 case MVT::v2i64: return fastEmit_AArch64ISD_CMLTz_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
588 default: return 0;
589 }
590}
591
592// FastEmit functions for AArch64ISD::DUP.
593
594unsigned fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v8i8_r(unsigned Op0, bool Op0IsKill) {
595 if ((Subtarget->hasNEON())) {
596 return fastEmitInst_r(AArch64::DUPv8i8gpr, &AArch64::FPR64RegClass, Op0, Op0IsKill);
597 }
598 return 0;
599}
600
601unsigned fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
602 if ((Subtarget->hasNEON())) {
603 return fastEmitInst_r(AArch64::DUPv16i8gpr, &AArch64::FPR128RegClass, Op0, Op0IsKill);
604 }
605 return 0;
606}
607
608unsigned fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v4i16_r(unsigned Op0, bool Op0IsKill) {
609 if ((Subtarget->hasNEON())) {
610 return fastEmitInst_r(AArch64::DUPv4i16gpr, &AArch64::FPR64RegClass, Op0, Op0IsKill);
611 }
612 return 0;
613}
614
615unsigned fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
616 if ((Subtarget->hasNEON())) {
617 return fastEmitInst_r(AArch64::DUPv8i16gpr, &AArch64::FPR128RegClass, Op0, Op0IsKill);
618 }
619 return 0;
620}
621
622unsigned fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v2i32_r(unsigned Op0, bool Op0IsKill) {
623 if ((Subtarget->hasNEON())) {
624 return fastEmitInst_r(AArch64::DUPv2i32gpr, &AArch64::FPR64RegClass, Op0, Op0IsKill);
625 }
626 return 0;
627}
628
629unsigned fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
630 if ((Subtarget->hasNEON())) {
631 return fastEmitInst_r(AArch64::DUPv4i32gpr, &AArch64::FPR128RegClass, Op0, Op0IsKill);
632 }
633 return 0;
634}
635
636unsigned fastEmit_AArch64ISD_DUP_MVT_i32_MVT_nxv16i8_r(unsigned Op0, bool Op0IsKill) {
637 if ((Subtarget->hasSVE())) {
638 return fastEmitInst_r(AArch64::DUP_ZR_B, &AArch64::ZPRRegClass, Op0, Op0IsKill);
639 }
640 return 0;
641}
642
643unsigned fastEmit_AArch64ISD_DUP_MVT_i32_MVT_nxv8i16_r(unsigned Op0, bool Op0IsKill) {
644 if ((Subtarget->hasSVE())) {
645 return fastEmitInst_r(AArch64::DUP_ZR_H, &AArch64::ZPRRegClass, Op0, Op0IsKill);
646 }
647 return 0;
648}
649
650unsigned fastEmit_AArch64ISD_DUP_MVT_i32_MVT_nxv4i32_r(unsigned Op0, bool Op0IsKill) {
651 if ((Subtarget->hasSVE())) {
652 return fastEmitInst_r(AArch64::DUP_ZR_S, &AArch64::ZPRRegClass, Op0, Op0IsKill);
653 }
654 return 0;
655}
656
657unsigned fastEmit_AArch64ISD_DUP_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
658switch (RetVT.SimpleTy) {
659 case MVT::v8i8: return fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v8i8_r(Op0, Op0IsKill);
660 case MVT::v16i8: return fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v16i8_r(Op0, Op0IsKill);
661 case MVT::v4i16: return fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v4i16_r(Op0, Op0IsKill);
662 case MVT::v8i16: return fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v8i16_r(Op0, Op0IsKill);
663 case MVT::v2i32: return fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v2i32_r(Op0, Op0IsKill);
664 case MVT::v4i32: return fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v4i32_r(Op0, Op0IsKill);
665 case MVT::nxv16i8: return fastEmit_AArch64ISD_DUP_MVT_i32_MVT_nxv16i8_r(Op0, Op0IsKill);
666 case MVT::nxv8i16: return fastEmit_AArch64ISD_DUP_MVT_i32_MVT_nxv8i16_r(Op0, Op0IsKill);
667 case MVT::nxv4i32: return fastEmit_AArch64ISD_DUP_MVT_i32_MVT_nxv4i32_r(Op0, Op0IsKill);
668 default: return 0;
669}
670}
671
672unsigned fastEmit_AArch64ISD_DUP_MVT_i64_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
673 if ((Subtarget->hasNEON())) {
674 return fastEmitInst_r(AArch64::DUPv2i64gpr, &AArch64::FPR128RegClass, Op0, Op0IsKill);
675 }
676 return 0;
677}
678
679unsigned fastEmit_AArch64ISD_DUP_MVT_i64_MVT_nxv2i64_r(unsigned Op0, bool Op0IsKill) {
680 if ((Subtarget->hasSVE())) {
681 return fastEmitInst_r(AArch64::DUP_ZR_D, &AArch64::ZPRRegClass, Op0, Op0IsKill);
682 }
683 return 0;
684}
685
686unsigned fastEmit_AArch64ISD_DUP_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
687switch (RetVT.SimpleTy) {
688 case MVT::v2i64: return fastEmit_AArch64ISD_DUP_MVT_i64_MVT_v2i64_r(Op0, Op0IsKill);
689 case MVT::nxv2i64: return fastEmit_AArch64ISD_DUP_MVT_i64_MVT_nxv2i64_r(Op0, Op0IsKill);
690 default: return 0;
691}
692}
693
694unsigned fastEmit_AArch64ISD_DUP_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
695 switch (VT.SimpleTy) {
696 case MVT::i32: return fastEmit_AArch64ISD_DUP_MVT_i32_r(RetVT, Op0, Op0IsKill);
697 case MVT::i64: return fastEmit_AArch64ISD_DUP_MVT_i64_r(RetVT, Op0, Op0IsKill);
698 default: return 0;
699 }
700}
701
702// FastEmit functions for AArch64ISD::FCMEQz.
703
704unsigned fastEmit_AArch64ISD_FCMEQz_MVT_v4f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
705 if (RetVT.SimpleTy != MVT::v4i16)
706 return 0;
707 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
708 return fastEmitInst_r(AArch64::FCMEQv4i16rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
709 }
710 return 0;
711}
712
713unsigned fastEmit_AArch64ISD_FCMEQz_MVT_v8f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
714 if (RetVT.SimpleTy != MVT::v8i16)
715 return 0;
716 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
717 return fastEmitInst_r(AArch64::FCMEQv8i16rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
718 }
719 return 0;
720}
721
722unsigned fastEmit_AArch64ISD_FCMEQz_MVT_v2f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
723 if (RetVT.SimpleTy != MVT::v2i32)
724 return 0;
725 if ((Subtarget->hasNEON())) {
726 return fastEmitInst_r(AArch64::FCMEQv2i32rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
727 }
728 return 0;
729}
730
731unsigned fastEmit_AArch64ISD_FCMEQz_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
732 if (RetVT.SimpleTy != MVT::v4i32)
733 return 0;
734 if ((Subtarget->hasNEON())) {
735 return fastEmitInst_r(AArch64::FCMEQv4i32rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
736 }
737 return 0;
738}
739
740unsigned fastEmit_AArch64ISD_FCMEQz_MVT_v1f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
741 if (RetVT.SimpleTy != MVT::v1i64)
742 return 0;
743 if ((Subtarget->hasNEON())) {
744 return fastEmitInst_r(AArch64::FCMEQv1i64rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
745 }
746 return 0;
747}
748
749unsigned fastEmit_AArch64ISD_FCMEQz_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
750 if (RetVT.SimpleTy != MVT::v2i64)
751 return 0;
752 if ((Subtarget->hasNEON())) {
753 return fastEmitInst_r(AArch64::FCMEQv2i64rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
754 }
755 return 0;
756}
757
758unsigned fastEmit_AArch64ISD_FCMEQz_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
759 switch (VT.SimpleTy) {
760 case MVT::v4f16: return fastEmit_AArch64ISD_FCMEQz_MVT_v4f16_r(RetVT, Op0, Op0IsKill);
761 case MVT::v8f16: return fastEmit_AArch64ISD_FCMEQz_MVT_v8f16_r(RetVT, Op0, Op0IsKill);
762 case MVT::v2f32: return fastEmit_AArch64ISD_FCMEQz_MVT_v2f32_r(RetVT, Op0, Op0IsKill);
763 case MVT::v4f32: return fastEmit_AArch64ISD_FCMEQz_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
764 case MVT::v1f64: return fastEmit_AArch64ISD_FCMEQz_MVT_v1f64_r(RetVT, Op0, Op0IsKill);
765 case MVT::v2f64: return fastEmit_AArch64ISD_FCMEQz_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
766 default: return 0;
767 }
768}
769
770// FastEmit functions for AArch64ISD::FCMGEz.
771
772unsigned fastEmit_AArch64ISD_FCMGEz_MVT_v4f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
773 if (RetVT.SimpleTy != MVT::v4i16)
774 return 0;
775 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
776 return fastEmitInst_r(AArch64::FCMGEv4i16rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
777 }
778 return 0;
779}
780
781unsigned fastEmit_AArch64ISD_FCMGEz_MVT_v8f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
782 if (RetVT.SimpleTy != MVT::v8i16)
783 return 0;
784 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
785 return fastEmitInst_r(AArch64::FCMGEv8i16rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
786 }
787 return 0;
788}
789
790unsigned fastEmit_AArch64ISD_FCMGEz_MVT_v2f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
791 if (RetVT.SimpleTy != MVT::v2i32)
792 return 0;
793 if ((Subtarget->hasNEON())) {
794 return fastEmitInst_r(AArch64::FCMGEv2i32rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
795 }
796 return 0;
797}
798
799unsigned fastEmit_AArch64ISD_FCMGEz_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
800 if (RetVT.SimpleTy != MVT::v4i32)
801 return 0;
802 if ((Subtarget->hasNEON())) {
803 return fastEmitInst_r(AArch64::FCMGEv4i32rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
804 }
805 return 0;
806}
807
808unsigned fastEmit_AArch64ISD_FCMGEz_MVT_v1f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
809 if (RetVT.SimpleTy != MVT::v1i64)
810 return 0;
811 if ((Subtarget->hasNEON())) {
812 return fastEmitInst_r(AArch64::FCMGEv1i64rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
813 }
814 return 0;
815}
816
817unsigned fastEmit_AArch64ISD_FCMGEz_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
818 if (RetVT.SimpleTy != MVT::v2i64)
819 return 0;
820 if ((Subtarget->hasNEON())) {
821 return fastEmitInst_r(AArch64::FCMGEv2i64rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
822 }
823 return 0;
824}
825
826unsigned fastEmit_AArch64ISD_FCMGEz_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
827 switch (VT.SimpleTy) {
828 case MVT::v4f16: return fastEmit_AArch64ISD_FCMGEz_MVT_v4f16_r(RetVT, Op0, Op0IsKill);
829 case MVT::v8f16: return fastEmit_AArch64ISD_FCMGEz_MVT_v8f16_r(RetVT, Op0, Op0IsKill);
830 case MVT::v2f32: return fastEmit_AArch64ISD_FCMGEz_MVT_v2f32_r(RetVT, Op0, Op0IsKill);
831 case MVT::v4f32: return fastEmit_AArch64ISD_FCMGEz_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
832 case MVT::v1f64: return fastEmit_AArch64ISD_FCMGEz_MVT_v1f64_r(RetVT, Op0, Op0IsKill);
833 case MVT::v2f64: return fastEmit_AArch64ISD_FCMGEz_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
834 default: return 0;
835 }
836}
837
838// FastEmit functions for AArch64ISD::FCMGTz.
839
840unsigned fastEmit_AArch64ISD_FCMGTz_MVT_v4f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
841 if (RetVT.SimpleTy != MVT::v4i16)
842 return 0;
843 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
844 return fastEmitInst_r(AArch64::FCMGTv4i16rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
845 }
846 return 0;
847}
848
849unsigned fastEmit_AArch64ISD_FCMGTz_MVT_v8f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
850 if (RetVT.SimpleTy != MVT::v8i16)
851 return 0;
852 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
853 return fastEmitInst_r(AArch64::FCMGTv8i16rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
854 }
855 return 0;
856}
857
858unsigned fastEmit_AArch64ISD_FCMGTz_MVT_v2f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
859 if (RetVT.SimpleTy != MVT::v2i32)
860 return 0;
861 if ((Subtarget->hasNEON())) {
862 return fastEmitInst_r(AArch64::FCMGTv2i32rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
863 }
864 return 0;
865}
866
867unsigned fastEmit_AArch64ISD_FCMGTz_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
868 if (RetVT.SimpleTy != MVT::v4i32)
869 return 0;
870 if ((Subtarget->hasNEON())) {
871 return fastEmitInst_r(AArch64::FCMGTv4i32rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
872 }
873 return 0;
874}
875
876unsigned fastEmit_AArch64ISD_FCMGTz_MVT_v1f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
877 if (RetVT.SimpleTy != MVT::v1i64)
878 return 0;
879 if ((Subtarget->hasNEON())) {
880 return fastEmitInst_r(AArch64::FCMGTv1i64rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
881 }
882 return 0;
883}
884
885unsigned fastEmit_AArch64ISD_FCMGTz_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
886 if (RetVT.SimpleTy != MVT::v2i64)
887 return 0;
888 if ((Subtarget->hasNEON())) {
889 return fastEmitInst_r(AArch64::FCMGTv2i64rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
890 }
891 return 0;
892}
893
894unsigned fastEmit_AArch64ISD_FCMGTz_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
895 switch (VT.SimpleTy) {
896 case MVT::v4f16: return fastEmit_AArch64ISD_FCMGTz_MVT_v4f16_r(RetVT, Op0, Op0IsKill);
897 case MVT::v8f16: return fastEmit_AArch64ISD_FCMGTz_MVT_v8f16_r(RetVT, Op0, Op0IsKill);
898 case MVT::v2f32: return fastEmit_AArch64ISD_FCMGTz_MVT_v2f32_r(RetVT, Op0, Op0IsKill);
899 case MVT::v4f32: return fastEmit_AArch64ISD_FCMGTz_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
900 case MVT::v1f64: return fastEmit_AArch64ISD_FCMGTz_MVT_v1f64_r(RetVT, Op0, Op0IsKill);
901 case MVT::v2f64: return fastEmit_AArch64ISD_FCMGTz_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
902 default: return 0;
903 }
904}
905
906// FastEmit functions for AArch64ISD::FCMLEz.
907
908unsigned fastEmit_AArch64ISD_FCMLEz_MVT_v4f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
909 if (RetVT.SimpleTy != MVT::v4i16)
910 return 0;
911 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
912 return fastEmitInst_r(AArch64::FCMLEv4i16rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
913 }
914 return 0;
915}
916
917unsigned fastEmit_AArch64ISD_FCMLEz_MVT_v8f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
918 if (RetVT.SimpleTy != MVT::v8i16)
919 return 0;
920 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
921 return fastEmitInst_r(AArch64::FCMLEv8i16rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
922 }
923 return 0;
924}
925
926unsigned fastEmit_AArch64ISD_FCMLEz_MVT_v2f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
927 if (RetVT.SimpleTy != MVT::v2i32)
928 return 0;
929 if ((Subtarget->hasNEON())) {
930 return fastEmitInst_r(AArch64::FCMLEv2i32rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
931 }
932 return 0;
933}
934
935unsigned fastEmit_AArch64ISD_FCMLEz_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
936 if (RetVT.SimpleTy != MVT::v4i32)
937 return 0;
938 if ((Subtarget->hasNEON())) {
939 return fastEmitInst_r(AArch64::FCMLEv4i32rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
940 }
941 return 0;
942}
943
944unsigned fastEmit_AArch64ISD_FCMLEz_MVT_v1f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
945 if (RetVT.SimpleTy != MVT::v1i64)
946 return 0;
947 if ((Subtarget->hasNEON())) {
948 return fastEmitInst_r(AArch64::FCMLEv1i64rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
949 }
950 return 0;
951}
952
953unsigned fastEmit_AArch64ISD_FCMLEz_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
954 if (RetVT.SimpleTy != MVT::v2i64)
955 return 0;
956 if ((Subtarget->hasNEON())) {
957 return fastEmitInst_r(AArch64::FCMLEv2i64rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
958 }
959 return 0;
960}
961
962unsigned fastEmit_AArch64ISD_FCMLEz_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
963 switch (VT.SimpleTy) {
964 case MVT::v4f16: return fastEmit_AArch64ISD_FCMLEz_MVT_v4f16_r(RetVT, Op0, Op0IsKill);
965 case MVT::v8f16: return fastEmit_AArch64ISD_FCMLEz_MVT_v8f16_r(RetVT, Op0, Op0IsKill);
966 case MVT::v2f32: return fastEmit_AArch64ISD_FCMLEz_MVT_v2f32_r(RetVT, Op0, Op0IsKill);
967 case MVT::v4f32: return fastEmit_AArch64ISD_FCMLEz_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
968 case MVT::v1f64: return fastEmit_AArch64ISD_FCMLEz_MVT_v1f64_r(RetVT, Op0, Op0IsKill);
969 case MVT::v2f64: return fastEmit_AArch64ISD_FCMLEz_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
970 default: return 0;
971 }
972}
973
974// FastEmit functions for AArch64ISD::FCMLTz.
975
976unsigned fastEmit_AArch64ISD_FCMLTz_MVT_v4f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
977 if (RetVT.SimpleTy != MVT::v4i16)
978 return 0;
979 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
980 return fastEmitInst_r(AArch64::FCMLTv4i16rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
981 }
982 return 0;
983}
984
985unsigned fastEmit_AArch64ISD_FCMLTz_MVT_v8f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
986 if (RetVT.SimpleTy != MVT::v8i16)
987 return 0;
988 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
989 return fastEmitInst_r(AArch64::FCMLTv8i16rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
990 }
991 return 0;
992}
993
994unsigned fastEmit_AArch64ISD_FCMLTz_MVT_v2f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
995 if (RetVT.SimpleTy != MVT::v2i32)
996 return 0;
997 if ((Subtarget->hasNEON())) {
998 return fastEmitInst_r(AArch64::FCMLTv2i32rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
999 }
1000 return 0;
1001}
1002
1003unsigned fastEmit_AArch64ISD_FCMLTz_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1004 if (RetVT.SimpleTy != MVT::v4i32)
1005 return 0;
1006 if ((Subtarget->hasNEON())) {
1007 return fastEmitInst_r(AArch64::FCMLTv4i32rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
1008 }
1009 return 0;
1010}
1011
1012unsigned fastEmit_AArch64ISD_FCMLTz_MVT_v1f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1013 if (RetVT.SimpleTy != MVT::v1i64)
1014 return 0;
1015 if ((Subtarget->hasNEON())) {
1016 return fastEmitInst_r(AArch64::FCMLTv1i64rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
1017 }
1018 return 0;
1019}
1020
1021unsigned fastEmit_AArch64ISD_FCMLTz_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1022 if (RetVT.SimpleTy != MVT::v2i64)
1023 return 0;
1024 if ((Subtarget->hasNEON())) {
1025 return fastEmitInst_r(AArch64::FCMLTv2i64rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
1026 }
1027 return 0;
1028}
1029
1030unsigned fastEmit_AArch64ISD_FCMLTz_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
1031 switch (VT.SimpleTy) {
1032 case MVT::v4f16: return fastEmit_AArch64ISD_FCMLTz_MVT_v4f16_r(RetVT, Op0, Op0IsKill);
1033 case MVT::v8f16: return fastEmit_AArch64ISD_FCMLTz_MVT_v8f16_r(RetVT, Op0, Op0IsKill);
1034 case MVT::v2f32: return fastEmit_AArch64ISD_FCMLTz_MVT_v2f32_r(RetVT, Op0, Op0IsKill);
1035 case MVT::v4f32: return fastEmit_AArch64ISD_FCMLTz_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
1036 case MVT::v1f64: return fastEmit_AArch64ISD_FCMLTz_MVT_v1f64_r(RetVT, Op0, Op0IsKill);
1037 case MVT::v2f64: return fastEmit_AArch64ISD_FCMLTz_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
1038 default: return 0;
1039 }
1040}
1041
1042// FastEmit functions for AArch64ISD::FRECPE.
1043
1044unsigned fastEmit_AArch64ISD_FRECPE_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1045 if (RetVT.SimpleTy != MVT::f32)
1046 return 0;
1047 return fastEmitInst_r(AArch64::FRECPEv1i32, &AArch64::FPR32RegClass, Op0, Op0IsKill);
1048}
1049
1050unsigned fastEmit_AArch64ISD_FRECPE_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1051 if (RetVT.SimpleTy != MVT::f64)
1052 return 0;
1053 return fastEmitInst_r(AArch64::FRECPEv1i64, &AArch64::FPR64RegClass, Op0, Op0IsKill);
1054}
1055
1056unsigned fastEmit_AArch64ISD_FRECPE_MVT_v2f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1057 if (RetVT.SimpleTy != MVT::v2f32)
1058 return 0;
1059 return fastEmitInst_r(AArch64::FRECPEv2f32, &AArch64::FPR64RegClass, Op0, Op0IsKill);
1060}
1061
1062unsigned fastEmit_AArch64ISD_FRECPE_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1063 if (RetVT.SimpleTy != MVT::v4f32)
1064 return 0;
1065 return fastEmitInst_r(AArch64::FRECPEv4f32, &AArch64::FPR128RegClass, Op0, Op0IsKill);
1066}
1067
1068unsigned fastEmit_AArch64ISD_FRECPE_MVT_v1f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1069 if (RetVT.SimpleTy != MVT::v1f64)
1070 return 0;
1071 return fastEmitInst_r(AArch64::FRECPEv1i64, &AArch64::FPR64RegClass, Op0, Op0IsKill);
1072}
1073
1074unsigned fastEmit_AArch64ISD_FRECPE_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1075 if (RetVT.SimpleTy != MVT::v2f64)
1076 return 0;
1077 return fastEmitInst_r(AArch64::FRECPEv2f64, &AArch64::FPR128RegClass, Op0, Op0IsKill);
1078}
1079
1080unsigned fastEmit_AArch64ISD_FRECPE_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
1081 switch (VT.SimpleTy) {
1082 case MVT::f32: return fastEmit_AArch64ISD_FRECPE_MVT_f32_r(RetVT, Op0, Op0IsKill);
1083 case MVT::f64: return fastEmit_AArch64ISD_FRECPE_MVT_f64_r(RetVT, Op0, Op0IsKill);
1084 case MVT::v2f32: return fastEmit_AArch64ISD_FRECPE_MVT_v2f32_r(RetVT, Op0, Op0IsKill);
1085 case MVT::v4f32: return fastEmit_AArch64ISD_FRECPE_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
1086 case MVT::v1f64: return fastEmit_AArch64ISD_FRECPE_MVT_v1f64_r(RetVT, Op0, Op0IsKill);
1087 case MVT::v2f64: return fastEmit_AArch64ISD_FRECPE_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
1088 default: return 0;
1089 }
1090}
1091
1092// FastEmit functions for AArch64ISD::FRSQRTE.
1093
1094unsigned fastEmit_AArch64ISD_FRSQRTE_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1095 if (RetVT.SimpleTy != MVT::f32)
1096 return 0;
1097 return fastEmitInst_r(AArch64::FRSQRTEv1i32, &AArch64::FPR32RegClass, Op0, Op0IsKill);
1098}
1099
1100unsigned fastEmit_AArch64ISD_FRSQRTE_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1101 if (RetVT.SimpleTy != MVT::f64)
1102 return 0;
1103 return fastEmitInst_r(AArch64::FRSQRTEv1i64, &AArch64::FPR64RegClass, Op0, Op0IsKill);
1104}
1105
1106unsigned fastEmit_AArch64ISD_FRSQRTE_MVT_v2f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1107 if (RetVT.SimpleTy != MVT::v2f32)
1108 return 0;
1109 return fastEmitInst_r(AArch64::FRSQRTEv2f32, &AArch64::FPR64RegClass, Op0, Op0IsKill);
1110}
1111
1112unsigned fastEmit_AArch64ISD_FRSQRTE_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1113 if (RetVT.SimpleTy != MVT::v4f32)
1114 return 0;
1115 return fastEmitInst_r(AArch64::FRSQRTEv4f32, &AArch64::FPR128RegClass, Op0, Op0IsKill);
1116}
1117
1118unsigned fastEmit_AArch64ISD_FRSQRTE_MVT_v1f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1119 if (RetVT.SimpleTy != MVT::v1f64)
1120 return 0;
1121 return fastEmitInst_r(AArch64::FRSQRTEv1i64, &AArch64::FPR64RegClass, Op0, Op0IsKill);
1122}
1123
1124unsigned fastEmit_AArch64ISD_FRSQRTE_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1125 if (RetVT.SimpleTy != MVT::v2f64)
1126 return 0;
1127 return fastEmitInst_r(AArch64::FRSQRTEv2f64, &AArch64::FPR128RegClass, Op0, Op0IsKill);
1128}
1129
1130unsigned fastEmit_AArch64ISD_FRSQRTE_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
1131 switch (VT.SimpleTy) {
1132 case MVT::f32: return fastEmit_AArch64ISD_FRSQRTE_MVT_f32_r(RetVT, Op0, Op0IsKill);
1133 case MVT::f64: return fastEmit_AArch64ISD_FRSQRTE_MVT_f64_r(RetVT, Op0, Op0IsKill);
1134 case MVT::v2f32: return fastEmit_AArch64ISD_FRSQRTE_MVT_v2f32_r(RetVT, Op0, Op0IsKill);
1135 case MVT::v4f32: return fastEmit_AArch64ISD_FRSQRTE_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
1136 case MVT::v1f64: return fastEmit_AArch64ISD_FRSQRTE_MVT_v1f64_r(RetVT, Op0, Op0IsKill);
1137 case MVT::v2f64: return fastEmit_AArch64ISD_FRSQRTE_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
1138 default: return 0;
1139 }
1140}
1141
1142// FastEmit functions for AArch64ISD::NEG.
1143
1144unsigned fastEmit_AArch64ISD_NEG_MVT_v8i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1145 if (RetVT.SimpleTy != MVT::v8i8)
1146 return 0;
1147 return fastEmitInst_r(AArch64::NEGv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill);
1148}
1149
1150unsigned fastEmit_AArch64ISD_NEG_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1151 if (RetVT.SimpleTy != MVT::v16i8)
1152 return 0;
1153 return fastEmitInst_r(AArch64::NEGv16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill);
1154}
1155
1156unsigned fastEmit_AArch64ISD_NEG_MVT_v4i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1157 if (RetVT.SimpleTy != MVT::v4i16)
1158 return 0;
1159 return fastEmitInst_r(AArch64::NEGv4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
1160}
1161
1162unsigned fastEmit_AArch64ISD_NEG_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1163 if (RetVT.SimpleTy != MVT::v8i16)
1164 return 0;
1165 return fastEmitInst_r(AArch64::NEGv8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
1166}
1167
1168unsigned fastEmit_AArch64ISD_NEG_MVT_v2i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1169 if (RetVT.SimpleTy != MVT::v2i32)
1170 return 0;
1171 return fastEmitInst_r(AArch64::NEGv2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill);
1172}
1173
1174unsigned fastEmit_AArch64ISD_NEG_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1175 if (RetVT.SimpleTy != MVT::v4i32)
1176 return 0;
1177 return fastEmitInst_r(AArch64::NEGv4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill);
1178}
1179
1180unsigned fastEmit_AArch64ISD_NEG_MVT_v1i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1181 if (RetVT.SimpleTy != MVT::v1i64)
1182 return 0;
1183 return fastEmitInst_r(AArch64::NEGv1i64, &AArch64::FPR64RegClass, Op0, Op0IsKill);
1184}
1185
1186unsigned fastEmit_AArch64ISD_NEG_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1187 if (RetVT.SimpleTy != MVT::v2i64)
1188 return 0;
1189 return fastEmitInst_r(AArch64::NEGv2i64, &AArch64::FPR128RegClass, Op0, Op0IsKill);
1190}
1191
1192unsigned fastEmit_AArch64ISD_NEG_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
1193 switch (VT.SimpleTy) {
1194 case MVT::v8i8: return fastEmit_AArch64ISD_NEG_MVT_v8i8_r(RetVT, Op0, Op0IsKill);
1195 case MVT::v16i8: return fastEmit_AArch64ISD_NEG_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
1196 case MVT::v4i16: return fastEmit_AArch64ISD_NEG_MVT_v4i16_r(RetVT, Op0, Op0IsKill);
1197 case MVT::v8i16: return fastEmit_AArch64ISD_NEG_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
1198 case MVT::v2i32: return fastEmit_AArch64ISD_NEG_MVT_v2i32_r(RetVT, Op0, Op0IsKill);
1199 case MVT::v4i32: return fastEmit_AArch64ISD_NEG_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
1200 case MVT::v1i64: return fastEmit_AArch64ISD_NEG_MVT_v1i64_r(RetVT, Op0, Op0IsKill);
1201 case MVT::v2i64: return fastEmit_AArch64ISD_NEG_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
1202 default: return 0;
1203 }
1204}
1205
1206// FastEmit functions for AArch64ISD::REV.
1207
1208unsigned fastEmit_AArch64ISD_REV_MVT_nxv2i1_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1209 if (RetVT.SimpleTy != MVT::nxv2i1)
1210 return 0;
1211 if ((Subtarget->hasSVE())) {
1212 return fastEmitInst_r(AArch64::REV_PP_D, &AArch64::PPRRegClass, Op0, Op0IsKill);
1213 }
1214 return 0;
1215}
1216
1217unsigned fastEmit_AArch64ISD_REV_MVT_nxv4i1_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1218 if (RetVT.SimpleTy != MVT::nxv4i1)
1219 return 0;
1220 if ((Subtarget->hasSVE())) {
1221 return fastEmitInst_r(AArch64::REV_PP_S, &AArch64::PPRRegClass, Op0, Op0IsKill);
1222 }
1223 return 0;
1224}
1225
1226unsigned fastEmit_AArch64ISD_REV_MVT_nxv8i1_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1227 if (RetVT.SimpleTy != MVT::nxv8i1)
1228 return 0;
1229 if ((Subtarget->hasSVE())) {
1230 return fastEmitInst_r(AArch64::REV_PP_H, &AArch64::PPRRegClass, Op0, Op0IsKill);
1231 }
1232 return 0;
1233}
1234
1235unsigned fastEmit_AArch64ISD_REV_MVT_nxv16i1_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1236 if (RetVT.SimpleTy != MVT::nxv16i1)
1237 return 0;
1238 if ((Subtarget->hasSVE())) {
1239 return fastEmitInst_r(AArch64::REV_PP_B, &AArch64::PPRRegClass, Op0, Op0IsKill);
1240 }
1241 return 0;
1242}
1243
1244unsigned fastEmit_AArch64ISD_REV_MVT_nxv16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1245 if (RetVT.SimpleTy != MVT::nxv16i8)
1246 return 0;
1247 if ((Subtarget->hasSVE())) {
1248 return fastEmitInst_r(AArch64::REV_ZZ_B, &AArch64::ZPRRegClass, Op0, Op0IsKill);
1249 }
1250 return 0;
1251}
1252
1253unsigned fastEmit_AArch64ISD_REV_MVT_nxv8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1254 if (RetVT.SimpleTy != MVT::nxv8i16)
1255 return 0;
1256 if ((Subtarget->hasSVE())) {
1257 return fastEmitInst_r(AArch64::REV_ZZ_H, &AArch64::ZPRRegClass, Op0, Op0IsKill);
1258 }
1259 return 0;
1260}
1261
1262unsigned fastEmit_AArch64ISD_REV_MVT_nxv4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1263 if (RetVT.SimpleTy != MVT::nxv4i32)
1264 return 0;
1265 if ((Subtarget->hasSVE())) {
1266 return fastEmitInst_r(AArch64::REV_ZZ_S, &AArch64::ZPRRegClass, Op0, Op0IsKill);
1267 }
1268 return 0;
1269}
1270
1271unsigned fastEmit_AArch64ISD_REV_MVT_nxv2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1272 if (RetVT.SimpleTy != MVT::nxv2i64)
1273 return 0;
1274 if ((Subtarget->hasSVE())) {
1275 return fastEmitInst_r(AArch64::REV_ZZ_D, &AArch64::ZPRRegClass, Op0, Op0IsKill);
1276 }
1277 return 0;
1278}
1279
1280unsigned fastEmit_AArch64ISD_REV_MVT_nxv8f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1281 if (RetVT.SimpleTy != MVT::nxv8f16)
1282 return 0;
1283 if ((Subtarget->hasSVE())) {
1284 return fastEmitInst_r(AArch64::REV_ZZ_H, &AArch64::ZPRRegClass, Op0, Op0IsKill);
1285 }
1286 return 0;
1287}
1288
1289unsigned fastEmit_AArch64ISD_REV_MVT_nxv8bf16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1290 if (RetVT.SimpleTy != MVT::nxv8bf16)
1291 return 0;
1292 if ((Subtarget->hasSVE())) {
1293 return fastEmitInst_r(AArch64::REV_ZZ_H, &AArch64::ZPRRegClass, Op0, Op0IsKill);
1294 }
1295 return 0;
1296}
1297
1298unsigned fastEmit_AArch64ISD_REV_MVT_nxv4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1299 if (RetVT.SimpleTy != MVT::nxv4f32)
1300 return 0;
1301 if ((Subtarget->hasSVE())) {
1302 return fastEmitInst_r(AArch64::REV_ZZ_S, &AArch64::ZPRRegClass, Op0, Op0IsKill);
1303 }
1304 return 0;
1305}
1306
1307unsigned fastEmit_AArch64ISD_REV_MVT_nxv2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1308 if (RetVT.SimpleTy != MVT::nxv2f64)
1309 return 0;
1310 if ((Subtarget->hasSVE())) {
1311 return fastEmitInst_r(AArch64::REV_ZZ_D, &AArch64::ZPRRegClass, Op0, Op0IsKill);
1312 }
1313 return 0;
1314}
1315
1316unsigned fastEmit_AArch64ISD_REV_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
1317 switch (VT.SimpleTy) {
1318 case MVT::nxv2i1: return fastEmit_AArch64ISD_REV_MVT_nxv2i1_r(RetVT, Op0, Op0IsKill);
1319 case MVT::nxv4i1: return fastEmit_AArch64ISD_REV_MVT_nxv4i1_r(RetVT, Op0, Op0IsKill);
1320 case MVT::nxv8i1: return fastEmit_AArch64ISD_REV_MVT_nxv8i1_r(RetVT, Op0, Op0IsKill);
1321 case MVT::nxv16i1: return fastEmit_AArch64ISD_REV_MVT_nxv16i1_r(RetVT, Op0, Op0IsKill);
1322 case MVT::nxv16i8: return fastEmit_AArch64ISD_REV_MVT_nxv16i8_r(RetVT, Op0, Op0IsKill);
1323 case MVT::nxv8i16: return fastEmit_AArch64ISD_REV_MVT_nxv8i16_r(RetVT, Op0, Op0IsKill);
1324 case MVT::nxv4i32: return fastEmit_AArch64ISD_REV_MVT_nxv4i32_r(RetVT, Op0, Op0IsKill);
1325 case MVT::nxv2i64: return fastEmit_AArch64ISD_REV_MVT_nxv2i64_r(RetVT, Op0, Op0IsKill);
1326 case MVT::nxv8f16: return fastEmit_AArch64ISD_REV_MVT_nxv8f16_r(RetVT, Op0, Op0IsKill);
1327 case MVT::nxv8bf16: return fastEmit_AArch64ISD_REV_MVT_nxv8bf16_r(RetVT, Op0, Op0IsKill);
1328 case MVT::nxv4f32: return fastEmit_AArch64ISD_REV_MVT_nxv4f32_r(RetVT, Op0, Op0IsKill);
1329 case MVT::nxv2f64: return fastEmit_AArch64ISD_REV_MVT_nxv2f64_r(RetVT, Op0, Op0IsKill);
1330 default: return 0;
1331 }
1332}
1333
1334// FastEmit functions for AArch64ISD::REV16.
1335
1336unsigned fastEmit_AArch64ISD_REV16_MVT_v8i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1337 if (RetVT.SimpleTy != MVT::v8i8)
1338 return 0;
1339 if ((Subtarget->hasNEON())) {
1340 return fastEmitInst_r(AArch64::REV16v8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill);
1341 }
1342 return 0;
1343}
1344
1345unsigned fastEmit_AArch64ISD_REV16_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1346 if (RetVT.SimpleTy != MVT::v16i8)
1347 return 0;
1348 if ((Subtarget->hasNEON())) {
1349 return fastEmitInst_r(AArch64::REV16v16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill);
1350 }
1351 return 0;
1352}
1353
1354unsigned fastEmit_AArch64ISD_REV16_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
1355 switch (VT.SimpleTy) {
1356 case MVT::v8i8: return fastEmit_AArch64ISD_REV16_MVT_v8i8_r(RetVT, Op0, Op0IsKill);
1357 case MVT::v16i8: return fastEmit_AArch64ISD_REV16_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
1358 default: return 0;
1359 }
1360}
1361
1362// FastEmit functions for AArch64ISD::REV32.
1363
1364unsigned fastEmit_AArch64ISD_REV32_MVT_v8i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1365 if (RetVT.SimpleTy != MVT::v8i8)
1366 return 0;
1367 if ((Subtarget->hasNEON())) {
1368 return fastEmitInst_r(AArch64::REV32v8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill);
1369 }
1370 return 0;
1371}
1372
1373unsigned fastEmit_AArch64ISD_REV32_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1374 if (RetVT.SimpleTy != MVT::v16i8)
1375 return 0;
1376 if ((Subtarget->hasNEON())) {
1377 return fastEmitInst_r(AArch64::REV32v16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill);
1378 }
1379 return 0;
1380}
1381
1382unsigned fastEmit_AArch64ISD_REV32_MVT_v4i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1383 if (RetVT.SimpleTy != MVT::v4i16)
1384 return 0;
1385 if ((Subtarget->hasNEON())) {
1386 return fastEmitInst_r(AArch64::REV32v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
1387 }
1388 return 0;
1389}
1390
1391unsigned fastEmit_AArch64ISD_REV32_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1392 if (RetVT.SimpleTy != MVT::v8i16)
1393 return 0;
1394 if ((Subtarget->hasNEON())) {
1395 return fastEmitInst_r(AArch64::REV32v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
1396 }
1397 return 0;
1398}
1399
1400unsigned fastEmit_AArch64ISD_REV32_MVT_v4f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1401 if (RetVT.SimpleTy != MVT::v4f16)
1402 return 0;
1403 return fastEmitInst_r(AArch64::REV32v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
1404}
1405
1406unsigned fastEmit_AArch64ISD_REV32_MVT_v8f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1407 if (RetVT.SimpleTy != MVT::v8f16)
1408 return 0;
1409 return fastEmitInst_r(AArch64::REV32v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
1410}
1411
1412unsigned fastEmit_AArch64ISD_REV32_MVT_v4bf16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1413 if (RetVT.SimpleTy != MVT::v4bf16)
1414 return 0;
1415 return fastEmitInst_r(AArch64::REV32v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
1416}
1417
1418unsigned fastEmit_AArch64ISD_REV32_MVT_v8bf16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1419 if (RetVT.SimpleTy != MVT::v8bf16)
1420 return 0;
1421 return fastEmitInst_r(AArch64::REV32v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
1422}
1423
1424unsigned fastEmit_AArch64ISD_REV32_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
1425 switch (VT.SimpleTy) {
1426 case MVT::v8i8: return fastEmit_AArch64ISD_REV32_MVT_v8i8_r(RetVT, Op0, Op0IsKill);
1427 case MVT::v16i8: return fastEmit_AArch64ISD_REV32_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
1428 case MVT::v4i16: return fastEmit_AArch64ISD_REV32_MVT_v4i16_r(RetVT, Op0, Op0IsKill);
1429 case MVT::v8i16: return fastEmit_AArch64ISD_REV32_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
1430 case MVT::v4f16: return fastEmit_AArch64ISD_REV32_MVT_v4f16_r(RetVT, Op0, Op0IsKill);
1431 case MVT::v8f16: return fastEmit_AArch64ISD_REV32_MVT_v8f16_r(RetVT, Op0, Op0IsKill);
1432 case MVT::v4bf16: return fastEmit_AArch64ISD_REV32_MVT_v4bf16_r(RetVT, Op0, Op0IsKill);
1433 case MVT::v8bf16: return fastEmit_AArch64ISD_REV32_MVT_v8bf16_r(RetVT, Op0, Op0IsKill);
1434 default: return 0;
1435 }
1436}
1437
1438// FastEmit functions for AArch64ISD::REV64.
1439
1440unsigned fastEmit_AArch64ISD_REV64_MVT_v8i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1441 if (RetVT.SimpleTy != MVT::v8i8)
1442 return 0;
1443 if ((Subtarget->hasNEON())) {
1444 return fastEmitInst_r(AArch64::REV64v8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill);
1445 }
1446 return 0;
1447}
1448
1449unsigned fastEmit_AArch64ISD_REV64_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1450 if (RetVT.SimpleTy != MVT::v16i8)
1451 return 0;
1452 if ((Subtarget->hasNEON())) {
1453 return fastEmitInst_r(AArch64::REV64v16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill);
1454 }
1455 return 0;
1456}
1457
1458unsigned fastEmit_AArch64ISD_REV64_MVT_v4i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1459 if (RetVT.SimpleTy != MVT::v4i16)
1460 return 0;
1461 if ((Subtarget->hasNEON())) {
1462 return fastEmitInst_r(AArch64::REV64v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
1463 }
1464 return 0;
1465}
1466
1467unsigned fastEmit_AArch64ISD_REV64_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1468 if (RetVT.SimpleTy != MVT::v8i16)
1469 return 0;
1470 if ((Subtarget->hasNEON())) {
1471 return fastEmitInst_r(AArch64::REV64v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
1472 }
1473 return 0;
1474}
1475
1476unsigned fastEmit_AArch64ISD_REV64_MVT_v2i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1477 if (RetVT.SimpleTy != MVT::v2i32)
1478 return 0;
1479 if ((Subtarget->hasNEON())) {
1480 return fastEmitInst_r(AArch64::REV64v2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill);
1481 }
1482 return 0;
1483}
1484
1485unsigned fastEmit_AArch64ISD_REV64_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1486 if (RetVT.SimpleTy != MVT::v4i32)
1487 return 0;
1488 if ((Subtarget->hasNEON())) {
1489 return fastEmitInst_r(AArch64::REV64v4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill);
1490 }
1491 return 0;
1492}
1493
1494unsigned fastEmit_AArch64ISD_REV64_MVT_v4f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1495 if (RetVT.SimpleTy != MVT::v4f16)
1496 return 0;
1497 return fastEmitInst_r(AArch64::REV64v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
1498}
1499
1500unsigned fastEmit_AArch64ISD_REV64_MVT_v8f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1501 if (RetVT.SimpleTy != MVT::v8f16)
1502 return 0;
1503 return fastEmitInst_r(AArch64::REV64v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
1504}
1505
1506unsigned fastEmit_AArch64ISD_REV64_MVT_v4bf16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1507 if (RetVT.SimpleTy != MVT::v4bf16)
1508 return 0;
1509 return fastEmitInst_r(AArch64::REV64v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
1510}
1511
1512unsigned fastEmit_AArch64ISD_REV64_MVT_v8bf16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1513 if (RetVT.SimpleTy != MVT::v8bf16)
1514 return 0;
1515 return fastEmitInst_r(AArch64::REV64v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
1516}
1517
1518unsigned fastEmit_AArch64ISD_REV64_MVT_v2f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1519 if (RetVT.SimpleTy != MVT::v2f32)
1520 return 0;
1521 return fastEmitInst_r(AArch64::REV64v2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill);
1522}
1523
1524unsigned fastEmit_AArch64ISD_REV64_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1525 if (RetVT.SimpleTy != MVT::v4f32)
1526 return 0;
1527 return fastEmitInst_r(AArch64::REV64v4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill);
1528}
1529
1530unsigned fastEmit_AArch64ISD_REV64_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
1531 switch (VT.SimpleTy) {
1532 case MVT::v8i8: return fastEmit_AArch64ISD_REV64_MVT_v8i8_r(RetVT, Op0, Op0IsKill);
1533 case MVT::v16i8: return fastEmit_AArch64ISD_REV64_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
1534 case MVT::v4i16: return fastEmit_AArch64ISD_REV64_MVT_v4i16_r(RetVT, Op0, Op0IsKill);
1535 case MVT::v8i16: return fastEmit_AArch64ISD_REV64_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
1536 case MVT::v2i32: return fastEmit_AArch64ISD_REV64_MVT_v2i32_r(RetVT, Op0, Op0IsKill);
1537 case MVT::v4i32: return fastEmit_AArch64ISD_REV64_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
1538 case MVT::v4f16: return fastEmit_AArch64ISD_REV64_MVT_v4f16_r(RetVT, Op0, Op0IsKill);
1539 case MVT::v8f16: return fastEmit_AArch64ISD_REV64_MVT_v8f16_r(RetVT, Op0, Op0IsKill);
1540 case MVT::v4bf16: return fastEmit_AArch64ISD_REV64_MVT_v4bf16_r(RetVT, Op0, Op0IsKill);
1541 case MVT::v8bf16: return fastEmit_AArch64ISD_REV64_MVT_v8bf16_r(RetVT, Op0, Op0IsKill);
1542 case MVT::v2f32: return fastEmit_AArch64ISD_REV64_MVT_v2f32_r(RetVT, Op0, Op0IsKill);
1543 case MVT::v4f32: return fastEmit_AArch64ISD_REV64_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
1544 default: return 0;
1545 }
1546}
1547
1548// FastEmit functions for AArch64ISD::SITOF.
1549
1550unsigned fastEmit_AArch64ISD_SITOF_MVT_f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1551 if (RetVT.SimpleTy != MVT::f16)
1552 return 0;
1553 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
1554 return fastEmitInst_r(AArch64::SCVTFv1i16, &AArch64::FPR16RegClass, Op0, Op0IsKill);
1555 }
1556 return 0;
1557}
1558
1559unsigned fastEmit_AArch64ISD_SITOF_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1560 if (RetVT.SimpleTy != MVT::f32)
1561 return 0;
1562 if ((Subtarget->hasNEON())) {
1563 return fastEmitInst_r(AArch64::SCVTFv1i32, &AArch64::FPR32RegClass, Op0, Op0IsKill);
1564 }
1565 return 0;
1566}
1567
1568unsigned fastEmit_AArch64ISD_SITOF_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1569 if (RetVT.SimpleTy != MVT::f64)
1570 return 0;
1571 if ((Subtarget->hasNEON())) {
1572 return fastEmitInst_r(AArch64::SCVTFv1i64, &AArch64::FPR64RegClass, Op0, Op0IsKill);
1573 }
1574 return 0;
1575}
1576
1577unsigned fastEmit_AArch64ISD_SITOF_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
1578 switch (VT.SimpleTy) {
1579 case MVT::f16: return fastEmit_AArch64ISD_SITOF_MVT_f16_r(RetVT, Op0, Op0IsKill);
1580 case MVT::f32: return fastEmit_AArch64ISD_SITOF_MVT_f32_r(RetVT, Op0, Op0IsKill);
1581 case MVT::f64: return fastEmit_AArch64ISD_SITOF_MVT_f64_r(RetVT, Op0, Op0IsKill);
1582 default: return 0;
1583 }
1584}
1585
1586// FastEmit functions for AArch64ISD::SUNPKHI.
1587
1588unsigned fastEmit_AArch64ISD_SUNPKHI_MVT_nxv16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1589 if (RetVT.SimpleTy != MVT::nxv8i16)
1590 return 0;
1591 if ((Subtarget->hasSVE())) {
1592 return fastEmitInst_r(AArch64::SUNPKHI_ZZ_H, &AArch64::ZPRRegClass, Op0, Op0IsKill);
1593 }
1594 return 0;
1595}
1596
1597unsigned fastEmit_AArch64ISD_SUNPKHI_MVT_nxv8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1598 if (RetVT.SimpleTy != MVT::nxv4i32)
1599 return 0;
1600 if ((Subtarget->hasSVE())) {
1601 return fastEmitInst_r(AArch64::SUNPKHI_ZZ_S, &AArch64::ZPRRegClass, Op0, Op0IsKill);
1602 }
1603 return 0;
1604}
1605
1606unsigned fastEmit_AArch64ISD_SUNPKHI_MVT_nxv4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1607 if (RetVT.SimpleTy != MVT::nxv2i64)
1608 return 0;
1609 if ((Subtarget->hasSVE())) {
1610 return fastEmitInst_r(AArch64::SUNPKHI_ZZ_D, &AArch64::ZPRRegClass, Op0, Op0IsKill);
1611 }
1612 return 0;
1613}
1614
1615unsigned fastEmit_AArch64ISD_SUNPKHI_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
1616 switch (VT.SimpleTy) {
1617 case MVT::nxv16i8: return fastEmit_AArch64ISD_SUNPKHI_MVT_nxv16i8_r(RetVT, Op0, Op0IsKill);
1618 case MVT::nxv8i16: return fastEmit_AArch64ISD_SUNPKHI_MVT_nxv8i16_r(RetVT, Op0, Op0IsKill);
1619 case MVT::nxv4i32: return fastEmit_AArch64ISD_SUNPKHI_MVT_nxv4i32_r(RetVT, Op0, Op0IsKill);
1620 default: return 0;
1621 }
1622}
1623
1624// FastEmit functions for AArch64ISD::SUNPKLO.
1625
1626unsigned fastEmit_AArch64ISD_SUNPKLO_MVT_nxv16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1627 if (RetVT.SimpleTy != MVT::nxv8i16)
1628 return 0;
1629 if ((Subtarget->hasSVE())) {
1630 return fastEmitInst_r(AArch64::SUNPKLO_ZZ_H, &AArch64::ZPRRegClass, Op0, Op0IsKill);
1631 }
1632 return 0;
1633}
1634
1635unsigned fastEmit_AArch64ISD_SUNPKLO_MVT_nxv8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1636 if (RetVT.SimpleTy != MVT::nxv4i32)
1637 return 0;
1638 if ((Subtarget->hasSVE())) {
1639 return fastEmitInst_r(AArch64::SUNPKLO_ZZ_S, &AArch64::ZPRRegClass, Op0, Op0IsKill);
1640 }
1641 return 0;
1642}
1643
1644unsigned fastEmit_AArch64ISD_SUNPKLO_MVT_nxv4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1645 if (RetVT.SimpleTy != MVT::nxv2i64)
1646 return 0;
1647 if ((Subtarget->hasSVE())) {
1648 return fastEmitInst_r(AArch64::SUNPKLO_ZZ_D, &AArch64::ZPRRegClass, Op0, Op0IsKill);
1649 }
1650 return 0;
1651}
1652
1653unsigned fastEmit_AArch64ISD_SUNPKLO_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
1654 switch (VT.SimpleTy) {
1655 case MVT::nxv16i8: return fastEmit_AArch64ISD_SUNPKLO_MVT_nxv16i8_r(RetVT, Op0, Op0IsKill);
1656 case MVT::nxv8i16: return fastEmit_AArch64ISD_SUNPKLO_MVT_nxv8i16_r(RetVT, Op0, Op0IsKill);
1657 case MVT::nxv4i32: return fastEmit_AArch64ISD_SUNPKLO_MVT_nxv4i32_r(RetVT, Op0, Op0IsKill);
1658 default: return 0;
1659 }
1660}
1661
1662// FastEmit functions for AArch64ISD::UITOF.
1663
1664unsigned fastEmit_AArch64ISD_UITOF_MVT_f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1665 if (RetVT.SimpleTy != MVT::f16)
1666 return 0;
1667 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
1668 return fastEmitInst_r(AArch64::UCVTFv1i16, &AArch64::FPR16RegClass, Op0, Op0IsKill);
1669 }
1670 return 0;
1671}
1672
1673unsigned fastEmit_AArch64ISD_UITOF_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1674 if (RetVT.SimpleTy != MVT::f32)
1675 return 0;
1676 if ((Subtarget->hasNEON())) {
1677 return fastEmitInst_r(AArch64::UCVTFv1i32, &AArch64::FPR32RegClass, Op0, Op0IsKill);
1678 }
1679 return 0;
1680}
1681
1682unsigned fastEmit_AArch64ISD_UITOF_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1683 if (RetVT.SimpleTy != MVT::f64)
1684 return 0;
1685 if ((Subtarget->hasNEON())) {
1686 return fastEmitInst_r(AArch64::UCVTFv1i64, &AArch64::FPR64RegClass, Op0, Op0IsKill);
1687 }
1688 return 0;
1689}
1690
1691unsigned fastEmit_AArch64ISD_UITOF_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
1692 switch (VT.SimpleTy) {
1693 case MVT::f16: return fastEmit_AArch64ISD_UITOF_MVT_f16_r(RetVT, Op0, Op0IsKill);
1694 case MVT::f32: return fastEmit_AArch64ISD_UITOF_MVT_f32_r(RetVT, Op0, Op0IsKill);
1695 case MVT::f64: return fastEmit_AArch64ISD_UITOF_MVT_f64_r(RetVT, Op0, Op0IsKill);
1696 default: return 0;
1697 }
1698}
1699
1700// FastEmit functions for AArch64ISD::UUNPKHI.
1701
1702unsigned fastEmit_AArch64ISD_UUNPKHI_MVT_nxv16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1703 if (RetVT.SimpleTy != MVT::nxv8i16)
1704 return 0;
1705 if ((Subtarget->hasSVE())) {
1706 return fastEmitInst_r(AArch64::UUNPKHI_ZZ_H, &AArch64::ZPRRegClass, Op0, Op0IsKill);
1707 }
1708 return 0;
1709}
1710
1711unsigned fastEmit_AArch64ISD_UUNPKHI_MVT_nxv8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1712 if (RetVT.SimpleTy != MVT::nxv4i32)
1713 return 0;
1714 if ((Subtarget->hasSVE())) {
1715 return fastEmitInst_r(AArch64::UUNPKHI_ZZ_S, &AArch64::ZPRRegClass, Op0, Op0IsKill);
1716 }
1717 return 0;
1718}
1719
1720unsigned fastEmit_AArch64ISD_UUNPKHI_MVT_nxv4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1721 if (RetVT.SimpleTy != MVT::nxv2i64)
1722 return 0;
1723 if ((Subtarget->hasSVE())) {
1724 return fastEmitInst_r(AArch64::UUNPKHI_ZZ_D, &AArch64::ZPRRegClass, Op0, Op0IsKill);
1725 }
1726 return 0;
1727}
1728
1729unsigned fastEmit_AArch64ISD_UUNPKHI_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
1730 switch (VT.SimpleTy) {
1731 case MVT::nxv16i8: return fastEmit_AArch64ISD_UUNPKHI_MVT_nxv16i8_r(RetVT, Op0, Op0IsKill);
1732 case MVT::nxv8i16: return fastEmit_AArch64ISD_UUNPKHI_MVT_nxv8i16_r(RetVT, Op0, Op0IsKill);
1733 case MVT::nxv4i32: return fastEmit_AArch64ISD_UUNPKHI_MVT_nxv4i32_r(RetVT, Op0, Op0IsKill);
1734 default: return 0;
1735 }
1736}
1737
1738// FastEmit functions for AArch64ISD::UUNPKLO.
1739
1740unsigned fastEmit_AArch64ISD_UUNPKLO_MVT_nxv16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1741 if (RetVT.SimpleTy != MVT::nxv8i16)
1742 return 0;
1743 if ((Subtarget->hasSVE())) {
1744 return fastEmitInst_r(AArch64::UUNPKLO_ZZ_H, &AArch64::ZPRRegClass, Op0, Op0IsKill);
1745 }
1746 return 0;
1747}
1748
1749unsigned fastEmit_AArch64ISD_UUNPKLO_MVT_nxv8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1750 if (RetVT.SimpleTy != MVT::nxv4i32)
1751 return 0;
1752 if ((Subtarget->hasSVE())) {
1753 return fastEmitInst_r(AArch64::UUNPKLO_ZZ_S, &AArch64::ZPRRegClass, Op0, Op0IsKill);
1754 }
1755 return 0;
1756}
1757
1758unsigned fastEmit_AArch64ISD_UUNPKLO_MVT_nxv4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1759 if (RetVT.SimpleTy != MVT::nxv2i64)
1760 return 0;
1761 if ((Subtarget->hasSVE())) {
1762 return fastEmitInst_r(AArch64::UUNPKLO_ZZ_D, &AArch64::ZPRRegClass, Op0, Op0IsKill);
1763 }
1764 return 0;
1765}
1766
1767unsigned fastEmit_AArch64ISD_UUNPKLO_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
1768 switch (VT.SimpleTy) {
1769 case MVT::nxv16i8: return fastEmit_AArch64ISD_UUNPKLO_MVT_nxv16i8_r(RetVT, Op0, Op0IsKill);
1770 case MVT::nxv8i16: return fastEmit_AArch64ISD_UUNPKLO_MVT_nxv8i16_r(RetVT, Op0, Op0IsKill);
1771 case MVT::nxv4i32: return fastEmit_AArch64ISD_UUNPKLO_MVT_nxv4i32_r(RetVT, Op0, Op0IsKill);
1772 default: return 0;
1773 }
1774}
1775
1776// FastEmit functions for ISD::ABS.
1777
1778unsigned fastEmit_ISD_ABS_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1779 if (RetVT.SimpleTy != MVT::i64)
1780 return 0;
1781 if ((Subtarget->hasNEON())) {
1782 return fastEmitInst_r(AArch64::ABSv1i64, &AArch64::FPR64RegClass, Op0, Op0IsKill);
1783 }
1784 return 0;
1785}
1786
1787unsigned fastEmit_ISD_ABS_MVT_v8i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1788 if (RetVT.SimpleTy != MVT::v8i8)
1789 return 0;
1790 if ((Subtarget->hasNEON())) {
1791 return fastEmitInst_r(AArch64::ABSv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill);
1792 }
1793 return 0;
1794}
1795
1796unsigned fastEmit_ISD_ABS_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1797 if (RetVT.SimpleTy != MVT::v16i8)
1798 return 0;
1799 if ((Subtarget->hasNEON())) {
1800 return fastEmitInst_r(AArch64::ABSv16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill);
1801 }
1802 return 0;
1803}
1804
1805unsigned fastEmit_ISD_ABS_MVT_v4i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1806 if (RetVT.SimpleTy != MVT::v4i16)
1807 return 0;
1808 if ((Subtarget->hasNEON())) {
1809 return fastEmitInst_r(AArch64::ABSv4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
1810 }
1811 return 0;
1812}
1813
1814unsigned fastEmit_ISD_ABS_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1815 if (RetVT.SimpleTy != MVT::v8i16)
1816 return 0;
1817 if ((Subtarget->hasNEON())) {
1818 return fastEmitInst_r(AArch64::ABSv8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
1819 }
1820 return 0;
1821}
1822
1823unsigned fastEmit_ISD_ABS_MVT_v2i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1824 if (RetVT.SimpleTy != MVT::v2i32)
1825 return 0;
1826 if ((Subtarget->hasNEON())) {
1827 return fastEmitInst_r(AArch64::ABSv2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill);
1828 }
1829 return 0;
1830}
1831
1832unsigned fastEmit_ISD_ABS_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1833 if (RetVT.SimpleTy != MVT::v4i32)
1834 return 0;
1835 if ((Subtarget->hasNEON())) {
1836 return fastEmitInst_r(AArch64::ABSv4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill);
1837 }
1838 return 0;
1839}
1840
1841unsigned fastEmit_ISD_ABS_MVT_v1i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1842 if (RetVT.SimpleTy != MVT::v1i64)
1843 return 0;
1844 if ((Subtarget->hasNEON())) {
1845 return fastEmitInst_r(AArch64::ABSv1i64, &AArch64::FPR64RegClass, Op0, Op0IsKill);
1846 }
1847 return 0;
1848}
1849
1850unsigned fastEmit_ISD_ABS_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1851 if (RetVT.SimpleTy != MVT::v2i64)
1852 return 0;
1853 if ((Subtarget->hasNEON())) {
1854 return fastEmitInst_r(AArch64::ABSv2i64, &AArch64::FPR128RegClass, Op0, Op0IsKill);
1855 }
1856 return 0;
1857}
1858
1859unsigned fastEmit_ISD_ABS_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
1860 switch (VT.SimpleTy) {
1861 case MVT::i64: return fastEmit_ISD_ABS_MVT_i64_r(RetVT, Op0, Op0IsKill);
1862 case MVT::v8i8: return fastEmit_ISD_ABS_MVT_v8i8_r(RetVT, Op0, Op0IsKill);
1863 case MVT::v16i8: return fastEmit_ISD_ABS_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
1864 case MVT::v4i16: return fastEmit_ISD_ABS_MVT_v4i16_r(RetVT, Op0, Op0IsKill);
1865 case MVT::v8i16: return fastEmit_ISD_ABS_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
1866 case MVT::v2i32: return fastEmit_ISD_ABS_MVT_v2i32_r(RetVT, Op0, Op0IsKill);
1867 case MVT::v4i32: return fastEmit_ISD_ABS_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
1868 case MVT::v1i64: return fastEmit_ISD_ABS_MVT_v1i64_r(RetVT, Op0, Op0IsKill);
1869 case MVT::v2i64: return fastEmit_ISD_ABS_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
1870 default: return 0;
1871 }
1872}
1873
1874// FastEmit functions for ISD::BITCAST.
1875
1876unsigned fastEmit_ISD_BITCAST_MVT_f64_MVT_v8i8_r(unsigned Op0, bool Op0IsKill) {
1877 if ((!Subtarget->isLittleEndian())) {
1878 return fastEmitInst_r(AArch64::REV64v8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill);
1879 }
1880 return 0;
1881}
1882
1883unsigned fastEmit_ISD_BITCAST_MVT_f64_MVT_v4i16_r(unsigned Op0, bool Op0IsKill) {
1884 if ((!Subtarget->isLittleEndian())) {
1885 return fastEmitInst_r(AArch64::REV64v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
1886 }
1887 return 0;
1888}
1889
1890unsigned fastEmit_ISD_BITCAST_MVT_f64_MVT_v2i32_r(unsigned Op0, bool Op0IsKill) {
1891 if ((!Subtarget->isLittleEndian())) {
1892 return fastEmitInst_r(AArch64::REV64v2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill);
1893 }
1894 return 0;
1895}
1896
1897unsigned fastEmit_ISD_BITCAST_MVT_f64_MVT_v4f16_r(unsigned Op0, bool Op0IsKill) {
1898 if ((!Subtarget->isLittleEndian())) {
1899 return fastEmitInst_r(AArch64::REV64v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
1900 }
1901 return 0;
1902}
1903
1904unsigned fastEmit_ISD_BITCAST_MVT_f64_MVT_v4bf16_r(unsigned Op0, bool Op0IsKill) {
1905 if ((!Subtarget->isLittleEndian())) {
1906 return fastEmitInst_r(AArch64::REV64v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
1907 }
1908 return 0;
1909}
1910
1911unsigned fastEmit_ISD_BITCAST_MVT_f64_MVT_v2f32_r(unsigned Op0, bool Op0IsKill) {
1912 if ((!Subtarget->isLittleEndian())) {
1913 return fastEmitInst_r(AArch64::REV64v2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill);
1914 }
1915 return 0;
1916}
1917
1918unsigned fastEmit_ISD_BITCAST_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1919switch (RetVT.SimpleTy) {
1920 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v8i8_r(Op0, Op0IsKill);
1921 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v4i16_r(Op0, Op0IsKill);
1922 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v2i32_r(Op0, Op0IsKill);
1923 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v4f16_r(Op0, Op0IsKill);
1924 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v4bf16_r(Op0, Op0IsKill);
1925 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v2f32_r(Op0, Op0IsKill);
1926 default: return 0;
1927}
1928}
1929
1930unsigned fastEmit_ISD_BITCAST_MVT_v8i8_MVT_f64_r(unsigned Op0, bool Op0IsKill) {
1931 if ((!Subtarget->isLittleEndian())) {
1932 return fastEmitInst_r(AArch64::REV64v8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill);
1933 }
1934 return 0;
1935}
1936
1937unsigned fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4i16_r(unsigned Op0, bool Op0IsKill) {
1938 if ((!Subtarget->isLittleEndian())) {
1939 return fastEmitInst_r(AArch64::REV16v8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill);
1940 }
1941 return 0;
1942}
1943
1944unsigned fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v2i32_r(unsigned Op0, bool Op0IsKill) {
1945 if ((!Subtarget->isLittleEndian())) {
1946 return fastEmitInst_r(AArch64::REV32v8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill);
1947 }
1948 return 0;
1949}
1950
1951unsigned fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v1i64_r(unsigned Op0, bool Op0IsKill) {
1952 if ((!Subtarget->isLittleEndian())) {
1953 return fastEmitInst_r(AArch64::REV64v8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill);
1954 }
1955 return 0;
1956}
1957
1958unsigned fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4f16_r(unsigned Op0, bool Op0IsKill) {
1959 if ((!Subtarget->isLittleEndian())) {
1960 return fastEmitInst_r(AArch64::REV16v8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill);
1961 }
1962 return 0;
1963}
1964
1965unsigned fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4bf16_r(unsigned Op0, bool Op0IsKill) {
1966 if ((!Subtarget->isLittleEndian())) {
1967 return fastEmitInst_r(AArch64::REV16v8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill);
1968 }
1969 return 0;
1970}
1971
1972unsigned fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v2f32_r(unsigned Op0, bool Op0IsKill) {
1973 if ((!Subtarget->isLittleEndian())) {
1974 return fastEmitInst_r(AArch64::REV32v8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill);
1975 }
1976 return 0;
1977}
1978
1979unsigned fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v1f64_r(unsigned Op0, bool Op0IsKill) {
1980 if ((!Subtarget->isLittleEndian())) {
1981 return fastEmitInst_r(AArch64::REV64v8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill);
1982 }
1983 return 0;
1984}
1985
1986unsigned fastEmit_ISD_BITCAST_MVT_v8i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1987switch (RetVT.SimpleTy) {
1988 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_f64_r(Op0, Op0IsKill);
1989 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4i16_r(Op0, Op0IsKill);
1990 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v2i32_r(Op0, Op0IsKill);
1991 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v1i64_r(Op0, Op0IsKill);
1992 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4f16_r(Op0, Op0IsKill);
1993 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4bf16_r(Op0, Op0IsKill);
1994 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v2f32_r(Op0, Op0IsKill);
1995 case MVT::v1f64: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v1f64_r(Op0, Op0IsKill);
1996 default: return 0;
1997}
1998}
1999
2000unsigned fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
2001 if ((!Subtarget->isLittleEndian())) {
2002 return fastEmitInst_r(AArch64::REV16v16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill);
2003 }
2004 return 0;
2005}
2006
2007unsigned fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
2008 if ((!Subtarget->isLittleEndian())) {
2009 return fastEmitInst_r(AArch64::REV32v16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill);
2010 }
2011 return 0;
2012}
2013
2014unsigned fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
2015 if ((!Subtarget->isLittleEndian())) {
2016 return fastEmitInst_r(AArch64::REV64v16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill);
2017 }
2018 return 0;
2019}
2020
2021unsigned fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8f16_r(unsigned Op0, bool Op0IsKill) {
2022 if ((!Subtarget->isLittleEndian())) {
2023 return fastEmitInst_r(AArch64::REV16v16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill);
2024 }
2025 return 0;
2026}
2027
2028unsigned fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8bf16_r(unsigned Op0, bool Op0IsKill) {
2029 if ((!Subtarget->isLittleEndian())) {
2030 return fastEmitInst_r(AArch64::REV16v16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill);
2031 }
2032 return 0;
2033}
2034
2035unsigned fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v4f32_r(unsigned Op0, bool Op0IsKill) {
2036 if ((!Subtarget->isLittleEndian())) {
2037 return fastEmitInst_r(AArch64::REV32v16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill);
2038 }
2039 return 0;
2040}
2041
2042unsigned fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v2f64_r(unsigned Op0, bool Op0IsKill) {
2043 if ((!Subtarget->isLittleEndian())) {
2044 return fastEmitInst_r(AArch64::REV64v16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill);
2045 }
2046 return 0;
2047}
2048
2049unsigned fastEmit_ISD_BITCAST_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2050switch (RetVT.SimpleTy) {
2051 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8i16_r(Op0, Op0IsKill);
2052 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v4i32_r(Op0, Op0IsKill);
2053 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v2i64_r(Op0, Op0IsKill);
2054 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8f16_r(Op0, Op0IsKill);
2055 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8bf16_r(Op0, Op0IsKill);
2056 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v4f32_r(Op0, Op0IsKill);
2057 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v2f64_r(Op0, Op0IsKill);
2058 default: return 0;
2059}
2060}
2061
2062unsigned fastEmit_ISD_BITCAST_MVT_v4i16_MVT_f64_r(unsigned Op0, bool Op0IsKill) {
2063 if ((!Subtarget->isLittleEndian())) {
2064 return fastEmitInst_r(AArch64::REV64v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
2065 }
2066 return 0;
2067}
2068
2069unsigned fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v8i8_r(unsigned Op0, bool Op0IsKill) {
2070 if ((!Subtarget->isLittleEndian())) {
2071 return fastEmitInst_r(AArch64::REV16v8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill);
2072 }
2073 return 0;
2074}
2075
2076unsigned fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v2i32_r(unsigned Op0, bool Op0IsKill) {
2077 if ((!Subtarget->isLittleEndian())) {
2078 return fastEmitInst_r(AArch64::REV32v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
2079 }
2080 return 0;
2081}
2082
2083unsigned fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v1i64_r(unsigned Op0, bool Op0IsKill) {
2084 if ((!Subtarget->isLittleEndian())) {
2085 return fastEmitInst_r(AArch64::REV64v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
2086 }
2087 return 0;
2088}
2089
2090unsigned fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v2f32_r(unsigned Op0, bool Op0IsKill) {
2091 if ((!Subtarget->isLittleEndian())) {
2092 return fastEmitInst_r(AArch64::REV32v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
2093 }
2094 return 0;
2095}
2096
2097unsigned fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v1f64_r(unsigned Op0, bool Op0IsKill) {
2098 if ((!Subtarget->isLittleEndian())) {
2099 return fastEmitInst_r(AArch64::REV64v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
2100 }
2101 return 0;
2102}
2103
2104unsigned fastEmit_ISD_BITCAST_MVT_v4i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2105switch (RetVT.SimpleTy) {
2106 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_f64_r(Op0, Op0IsKill);
2107 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v8i8_r(Op0, Op0IsKill);
2108 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v2i32_r(Op0, Op0IsKill);
2109 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v1i64_r(Op0, Op0IsKill);
2110 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v2f32_r(Op0, Op0IsKill);
2111 case MVT::v1f64: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v1f64_r(Op0, Op0IsKill);
2112 default: return 0;
2113}
2114}
2115
2116unsigned fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
2117 if ((!Subtarget->isLittleEndian())) {
2118 return fastEmitInst_r(AArch64::REV16v16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill);
2119 }
2120 return 0;
2121}
2122
2123unsigned fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
2124 if ((!Subtarget->isLittleEndian())) {
2125 return fastEmitInst_r(AArch64::REV32v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
2126 }
2127 return 0;
2128}
2129
2130unsigned fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
2131 if ((!Subtarget->isLittleEndian())) {
2132 return fastEmitInst_r(AArch64::REV64v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
2133 }
2134 return 0;
2135}
2136
2137unsigned fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v4f32_r(unsigned Op0, bool Op0IsKill) {
2138 if ((!Subtarget->isLittleEndian())) {
2139 return fastEmitInst_r(AArch64::REV32v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
2140 }
2141 return 0;
2142}
2143
2144unsigned fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v2f64_r(unsigned Op0, bool Op0IsKill) {
2145 if ((!Subtarget->isLittleEndian())) {
2146 return fastEmitInst_r(AArch64::REV64v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
2147 }
2148 return 0;
2149}
2150
2151unsigned fastEmit_ISD_BITCAST_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2152switch (RetVT.SimpleTy) {
2153 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v16i8_r(Op0, Op0IsKill);
2154 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v4i32_r(Op0, Op0IsKill);
2155 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v2i64_r(Op0, Op0IsKill);
2156 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v4f32_r(Op0, Op0IsKill);
2157 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v2f64_r(Op0, Op0IsKill);
2158 default: return 0;
2159}
2160}
2161
2162unsigned fastEmit_ISD_BITCAST_MVT_v2i32_MVT_f64_r(unsigned Op0, bool Op0IsKill) {
2163 if ((!Subtarget->isLittleEndian())) {
2164 return fastEmitInst_r(AArch64::REV64v2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill);
2165 }
2166 return 0;
2167}
2168
2169unsigned fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v8i8_r(unsigned Op0, bool Op0IsKill) {
2170 if ((!Subtarget->isLittleEndian())) {
2171 return fastEmitInst_r(AArch64::REV32v8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill);
2172 }
2173 return 0;
2174}
2175
2176unsigned fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4i16_r(unsigned Op0, bool Op0IsKill) {
2177 if ((!Subtarget->isLittleEndian())) {
2178 return fastEmitInst_r(AArch64::REV32v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
2179 }
2180 return 0;
2181}
2182
2183unsigned fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v1i64_r(unsigned Op0, bool Op0IsKill) {
2184 if ((!Subtarget->isLittleEndian())) {
2185 return fastEmitInst_r(AArch64::REV64v2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill);
2186 }
2187 return 0;
2188}
2189
2190unsigned fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4f16_r(unsigned Op0, bool Op0IsKill) {
2191 if ((!Subtarget->isLittleEndian())) {
2192 return fastEmitInst_r(AArch64::REV32v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
2193 }
2194 return 0;
2195}
2196
2197unsigned fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4bf16_r(unsigned Op0, bool Op0IsKill) {
2198 if ((!Subtarget->isLittleEndian())) {
2199 return fastEmitInst_r(AArch64::REV32v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
2200 }
2201 return 0;
2202}
2203
2204unsigned fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v1f64_r(unsigned Op0, bool Op0IsKill) {
2205 if ((!Subtarget->isLittleEndian())) {
2206 return fastEmitInst_r(AArch64::REV64v2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill);
2207 }
2208 return 0;
2209}
2210
2211unsigned fastEmit_ISD_BITCAST_MVT_v2i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2212switch (RetVT.SimpleTy) {
2213 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_f64_r(Op0, Op0IsKill);
2214 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v8i8_r(Op0, Op0IsKill);
2215 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4i16_r(Op0, Op0IsKill);
2216 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v1i64_r(Op0, Op0IsKill);
2217 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4f16_r(Op0, Op0IsKill);
2218 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4bf16_r(Op0, Op0IsKill);
2219 case MVT::v1f64: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v1f64_r(Op0, Op0IsKill);
2220 default: return 0;
2221}
2222}
2223
2224unsigned fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
2225 if ((!Subtarget->isLittleEndian())) {
2226 return fastEmitInst_r(AArch64::REV32v16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill);
2227 }
2228 return 0;
2229}
2230
2231unsigned fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
2232 if ((!Subtarget->isLittleEndian())) {
2233 return fastEmitInst_r(AArch64::REV32v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
2234 }
2235 return 0;
2236}
2237
2238unsigned fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
2239 if ((!Subtarget->isLittleEndian())) {
2240 return fastEmitInst_r(AArch64::REV64v4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill);
2241 }
2242 return 0;
2243}
2244
2245unsigned fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8f16_r(unsigned Op0, bool Op0IsKill) {
2246 if ((!Subtarget->isLittleEndian())) {
2247 return fastEmitInst_r(AArch64::REV32v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
2248 }
2249 return 0;
2250}
2251
2252unsigned fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8bf16_r(unsigned Op0, bool Op0IsKill) {
2253 if ((!Subtarget->isLittleEndian())) {
2254 return fastEmitInst_r(AArch64::REV32v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
2255 }
2256 return 0;
2257}
2258
2259unsigned fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v2f64_r(unsigned Op0, bool Op0IsKill) {
2260 if ((!Subtarget->isLittleEndian())) {
2261 return fastEmitInst_r(AArch64::REV64v4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill);
2262 }
2263 return 0;
2264}
2265
2266unsigned fastEmit_ISD_BITCAST_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2267switch (RetVT.SimpleTy) {
2268 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v16i8_r(Op0, Op0IsKill);
2269 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8i16_r(Op0, Op0IsKill);
2270 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v2i64_r(Op0, Op0IsKill);
2271 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8f16_r(Op0, Op0IsKill);
2272 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8bf16_r(Op0, Op0IsKill);
2273 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v2f64_r(Op0, Op0IsKill);
2274 default: return 0;
2275}
2276}
2277
2278unsigned fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v8i8_r(unsigned Op0, bool Op0IsKill) {
2279 if ((!Subtarget->isLittleEndian())) {
2280 return fastEmitInst_r(AArch64::REV64v8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill);
2281 }
2282 return 0;
2283}
2284
2285unsigned fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4i16_r(unsigned Op0, bool Op0IsKill) {
2286 if ((!Subtarget->isLittleEndian())) {
2287 return fastEmitInst_r(AArch64::REV64v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
2288 }
2289 return 0;
2290}
2291
2292unsigned fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v2i32_r(unsigned Op0, bool Op0IsKill) {
2293 if ((!Subtarget->isLittleEndian())) {
2294 return fastEmitInst_r(AArch64::REV64v2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill);
2295 }
2296 return 0;
2297}
2298
2299unsigned fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4f16_r(unsigned Op0, bool Op0IsKill) {
2300 if ((!Subtarget->isLittleEndian())) {
2301 return fastEmitInst_r(AArch64::REV64v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
2302 }
2303 return 0;
2304}
2305
2306unsigned fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4bf16_r(unsigned Op0, bool Op0IsKill) {
2307 if ((!Subtarget->isLittleEndian())) {
2308 return fastEmitInst_r(AArch64::REV64v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
2309 }
2310 return 0;
2311}
2312
2313unsigned fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v2f32_r(unsigned Op0, bool Op0IsKill) {
2314 if ((!Subtarget->isLittleEndian())) {
2315 return fastEmitInst_r(AArch64::REV64v2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill);
2316 }
2317 return 0;
2318}
2319
2320unsigned fastEmit_ISD_BITCAST_MVT_v1i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2321switch (RetVT.SimpleTy) {
2322 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v8i8_r(Op0, Op0IsKill);
2323 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4i16_r(Op0, Op0IsKill);
2324 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v2i32_r(Op0, Op0IsKill);
2325 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4f16_r(Op0, Op0IsKill);
2326 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4bf16_r(Op0, Op0IsKill);
2327 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v2f32_r(Op0, Op0IsKill);
2328 default: return 0;
2329}
2330}
2331
2332unsigned fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
2333 if ((!Subtarget->isLittleEndian())) {
2334 return fastEmitInst_r(AArch64::REV64v16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill);
2335 }
2336 return 0;
2337}
2338
2339unsigned fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
2340 if ((!Subtarget->isLittleEndian())) {
2341 return fastEmitInst_r(AArch64::REV64v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
2342 }
2343 return 0;
2344}
2345
2346unsigned fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
2347 if ((!Subtarget->isLittleEndian())) {
2348 return fastEmitInst_r(AArch64::REV64v4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill);
2349 }
2350 return 0;
2351}
2352
2353unsigned fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8f16_r(unsigned Op0, bool Op0IsKill) {
2354 if ((!Subtarget->isLittleEndian())) {
2355 return fastEmitInst_r(AArch64::REV64v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
2356 }
2357 return 0;
2358}
2359
2360unsigned fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8bf16_r(unsigned Op0, bool Op0IsKill) {
2361 if ((!Subtarget->isLittleEndian())) {
2362 return fastEmitInst_r(AArch64::REV64v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
2363 }
2364 return 0;
2365}
2366
2367unsigned fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v4f32_r(unsigned Op0, bool Op0IsKill) {
2368 if ((!Subtarget->isLittleEndian())) {
2369 return fastEmitInst_r(AArch64::REV64v4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill);
2370 }
2371 return 0;
2372}
2373
2374unsigned fastEmit_ISD_BITCAST_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2375switch (RetVT.SimpleTy) {
2376 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v16i8_r(Op0, Op0IsKill);
2377 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8i16_r(Op0, Op0IsKill);
2378 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v4i32_r(Op0, Op0IsKill);
2379 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8f16_r(Op0, Op0IsKill);
2380 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8bf16_r(Op0, Op0IsKill);
2381 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v4f32_r(Op0, Op0IsKill);
2382 default: return 0;
2383}
2384}
2385
2386unsigned fastEmit_ISD_BITCAST_MVT_v4f16_MVT_f64_r(unsigned Op0, bool Op0IsKill) {
2387 if ((!Subtarget->isLittleEndian())) {
2388 return fastEmitInst_r(AArch64::REV64v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
2389 }
2390 return 0;
2391}
2392
2393unsigned fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v8i8_r(unsigned Op0, bool Op0IsKill) {
2394 if ((!Subtarget->isLittleEndian())) {
2395 return fastEmitInst_r(AArch64::REV16v8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill);
2396 }
2397 return 0;
2398}
2399
2400unsigned fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v2i32_r(unsigned Op0, bool Op0IsKill) {
2401 if ((!Subtarget->isLittleEndian())) {
2402 return fastEmitInst_r(AArch64::REV32v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
2403 }
2404 return 0;
2405}
2406
2407unsigned fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v1i64_r(unsigned Op0, bool Op0IsKill) {
2408 if ((!Subtarget->isLittleEndian())) {
2409 return fastEmitInst_r(AArch64::REV64v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
2410 }
2411 return 0;
2412}
2413
2414unsigned fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v2f32_r(unsigned Op0, bool Op0IsKill) {
2415 if ((!Subtarget->isLittleEndian())) {
2416 return fastEmitInst_r(AArch64::REV32v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
2417 }
2418 return 0;
2419}
2420
2421unsigned fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v1f64_r(unsigned Op0, bool Op0IsKill) {
2422 if ((!Subtarget->isLittleEndian())) {
2423 return fastEmitInst_r(AArch64::REV64v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
2424 }
2425 return 0;
2426}
2427
2428unsigned fastEmit_ISD_BITCAST_MVT_v4f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2429switch (RetVT.SimpleTy) {
2430 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_f64_r(Op0, Op0IsKill);
2431 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v8i8_r(Op0, Op0IsKill);
2432 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v2i32_r(Op0, Op0IsKill);
2433 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v1i64_r(Op0, Op0IsKill);
2434 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v2f32_r(Op0, Op0IsKill);
2435 case MVT::v1f64: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v1f64_r(Op0, Op0IsKill);
2436 default: return 0;
2437}
2438}
2439
2440unsigned fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
2441 if ((!Subtarget->isLittleEndian())) {
2442 return fastEmitInst_r(AArch64::REV16v16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill);
2443 }
2444 return 0;
2445}
2446
2447unsigned fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
2448 if ((!Subtarget->isLittleEndian())) {
2449 return fastEmitInst_r(AArch64::REV32v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
2450 }
2451 return 0;
2452}
2453
2454unsigned fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
2455 if ((!Subtarget->isLittleEndian())) {
2456 return fastEmitInst_r(AArch64::REV64v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
2457 }
2458 return 0;
2459}
2460
2461unsigned fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v4f32_r(unsigned Op0, bool Op0IsKill) {
2462 if ((!Subtarget->isLittleEndian())) {
2463 return fastEmitInst_r(AArch64::REV32v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
2464 }
2465 return 0;
2466}
2467
2468unsigned fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v2f64_r(unsigned Op0, bool Op0IsKill) {
2469 if ((!Subtarget->isLittleEndian())) {
2470 return fastEmitInst_r(AArch64::REV64v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
2471 }
2472 return 0;
2473}
2474
2475unsigned fastEmit_ISD_BITCAST_MVT_v8f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2476switch (RetVT.SimpleTy) {
2477 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v16i8_r(Op0, Op0IsKill);
2478 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v4i32_r(Op0, Op0IsKill);
2479 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v2i64_r(Op0, Op0IsKill);
2480 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v4f32_r(Op0, Op0IsKill);
2481 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v2f64_r(Op0, Op0IsKill);
2482 default: return 0;
2483}
2484}
2485
2486unsigned fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_f64_r(unsigned Op0, bool Op0IsKill) {
2487 if ((!Subtarget->isLittleEndian())) {
2488 return fastEmitInst_r(AArch64::REV64v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
2489 }
2490 return 0;
2491}
2492
2493unsigned fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v8i8_r(unsigned Op0, bool Op0IsKill) {
2494 if ((!Subtarget->isLittleEndian())) {
2495 return fastEmitInst_r(AArch64::REV16v8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill);
2496 }
2497 return 0;
2498}
2499
2500unsigned fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v2i32_r(unsigned Op0, bool Op0IsKill) {
2501 if ((!Subtarget->isLittleEndian())) {
2502 return fastEmitInst_r(AArch64::REV32v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
2503 }
2504 return 0;
2505}
2506
2507unsigned fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v1i64_r(unsigned Op0, bool Op0IsKill) {
2508 if ((!Subtarget->isLittleEndian())) {
2509 return fastEmitInst_r(AArch64::REV64v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
2510 }
2511 return 0;
2512}
2513
2514unsigned fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v2f32_r(unsigned Op0, bool Op0IsKill) {
2515 if ((!Subtarget->isLittleEndian())) {
2516 return fastEmitInst_r(AArch64::REV32v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
2517 }
2518 return 0;
2519}
2520
2521unsigned fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v1f64_r(unsigned Op0, bool Op0IsKill) {
2522 if ((!Subtarget->isLittleEndian())) {
2523 return fastEmitInst_r(AArch64::REV64v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
2524 }
2525 return 0;
2526}
2527
2528unsigned fastEmit_ISD_BITCAST_MVT_v4bf16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2529switch (RetVT.SimpleTy) {
2530 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_f64_r(Op0, Op0IsKill);
2531 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v8i8_r(Op0, Op0IsKill);
2532 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v2i32_r(Op0, Op0IsKill);
2533 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v1i64_r(Op0, Op0IsKill);
2534 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v2f32_r(Op0, Op0IsKill);
2535 case MVT::v1f64: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v1f64_r(Op0, Op0IsKill);
2536 default: return 0;
2537}
2538}
2539
2540unsigned fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
2541 if ((!Subtarget->isLittleEndian())) {
2542 return fastEmitInst_r(AArch64::REV16v16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill);
2543 }
2544 return 0;
2545}
2546
2547unsigned fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
2548 if ((!Subtarget->isLittleEndian())) {
2549 return fastEmitInst_r(AArch64::REV32v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
2550 }
2551 return 0;
2552}
2553
2554unsigned fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
2555 if ((!Subtarget->isLittleEndian())) {
2556 return fastEmitInst_r(AArch64::REV64v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
2557 }
2558 return 0;
2559}
2560
2561unsigned fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v4f32_r(unsigned Op0, bool Op0IsKill) {
2562 if ((!Subtarget->isLittleEndian())) {
2563 return fastEmitInst_r(AArch64::REV32v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
2564 }
2565 return 0;
2566}
2567
2568unsigned fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v2f64_r(unsigned Op0, bool Op0IsKill) {
2569 if ((!Subtarget->isLittleEndian())) {
2570 return fastEmitInst_r(AArch64::REV64v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
2571 }
2572 return 0;
2573}
2574
2575unsigned fastEmit_ISD_BITCAST_MVT_v8bf16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2576switch (RetVT.SimpleTy) {
2577 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v16i8_r(Op0, Op0IsKill);
2578 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v4i32_r(Op0, Op0IsKill);
2579 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v2i64_r(Op0, Op0IsKill);
2580 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v4f32_r(Op0, Op0IsKill);
2581 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v2f64_r(Op0, Op0IsKill);
2582 default: return 0;
2583}
2584}
2585
2586unsigned fastEmit_ISD_BITCAST_MVT_v2f32_MVT_f64_r(unsigned Op0, bool Op0IsKill) {
2587 if ((!Subtarget->isLittleEndian())) {
2588 return fastEmitInst_r(AArch64::REV64v2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill);
2589 }
2590 return 0;
2591}
2592
2593unsigned fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v8i8_r(unsigned Op0, bool Op0IsKill) {
2594 if ((!Subtarget->isLittleEndian())) {
2595 return fastEmitInst_r(AArch64::REV32v8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill);
2596 }
2597 return 0;
2598}
2599
2600unsigned fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4i16_r(unsigned Op0, bool Op0IsKill) {
2601 if ((!Subtarget->isLittleEndian())) {
2602 return fastEmitInst_r(AArch64::REV32v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
2603 }
2604 return 0;
2605}
2606
2607unsigned fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v1i64_r(unsigned Op0, bool Op0IsKill) {
2608 if ((!Subtarget->isLittleEndian())) {
2609 return fastEmitInst_r(AArch64::REV64v2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill);
2610 }
2611 return 0;
2612}
2613
2614unsigned fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4f16_r(unsigned Op0, bool Op0IsKill) {
2615 if ((!Subtarget->isLittleEndian())) {
2616 return fastEmitInst_r(AArch64::REV32v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
2617 }
2618 return 0;
2619}
2620
2621unsigned fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4bf16_r(unsigned Op0, bool Op0IsKill) {
2622 if ((!Subtarget->isLittleEndian())) {
2623 return fastEmitInst_r(AArch64::REV32v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
2624 }
2625 return 0;
2626}
2627
2628unsigned fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v1f64_r(unsigned Op0, bool Op0IsKill) {
2629 if ((!Subtarget->isLittleEndian())) {
2630 return fastEmitInst_r(AArch64::REV64v2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill);
2631 }
2632 return 0;
2633}
2634
2635unsigned fastEmit_ISD_BITCAST_MVT_v2f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2636switch (RetVT.SimpleTy) {
2637 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_f64_r(Op0, Op0IsKill);
2638 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v8i8_r(Op0, Op0IsKill);
2639 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4i16_r(Op0, Op0IsKill);
2640 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v1i64_r(Op0, Op0IsKill);
2641 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4f16_r(Op0, Op0IsKill);
2642 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4bf16_r(Op0, Op0IsKill);
2643 case MVT::v1f64: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v1f64_r(Op0, Op0IsKill);
2644 default: return 0;
2645}
2646}
2647
2648unsigned fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
2649 if ((!Subtarget->isLittleEndian())) {
2650 return fastEmitInst_r(AArch64::REV32v16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill);
2651 }
2652 return 0;
2653}
2654
2655unsigned fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
2656 if ((!Subtarget->isLittleEndian())) {
2657 return fastEmitInst_r(AArch64::REV32v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
2658 }
2659 return 0;
2660}
2661
2662unsigned fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
2663 if ((!Subtarget->isLittleEndian())) {
2664 return fastEmitInst_r(AArch64::REV64v4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill);
2665 }
2666 return 0;
2667}
2668
2669unsigned fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8f16_r(unsigned Op0, bool Op0IsKill) {
2670 if ((!Subtarget->isLittleEndian())) {
2671 return fastEmitInst_r(AArch64::REV32v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
2672 }
2673 return 0;
2674}
2675
2676unsigned fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8bf16_r(unsigned Op0, bool Op0IsKill) {
2677 if ((!Subtarget->isLittleEndian())) {
2678 return fastEmitInst_r(AArch64::REV32v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
2679 }
2680 return 0;
2681}
2682
2683unsigned fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v2f64_r(unsigned Op0, bool Op0IsKill) {
2684 if ((!Subtarget->isLittleEndian())) {
2685 return fastEmitInst_r(AArch64::REV64v4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill);
2686 }
2687 return 0;
2688}
2689
2690unsigned fastEmit_ISD_BITCAST_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2691switch (RetVT.SimpleTy) {
2692 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v16i8_r(Op0, Op0IsKill);
2693 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8i16_r(Op0, Op0IsKill);
2694 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v2i64_r(Op0, Op0IsKill);
2695 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8f16_r(Op0, Op0IsKill);
2696 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8bf16_r(Op0, Op0IsKill);
2697 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v2f64_r(Op0, Op0IsKill);
2698 default: return 0;
2699}
2700}
2701
2702unsigned fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v8i8_r(unsigned Op0, bool Op0IsKill) {
2703 if ((!Subtarget->isLittleEndian())) {
2704 return fastEmitInst_r(AArch64::REV64v8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill);
2705 }
2706 return 0;
2707}
2708
2709unsigned fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v4i16_r(unsigned Op0, bool Op0IsKill) {
2710 if ((!Subtarget->isLittleEndian())) {
2711 return fastEmitInst_r(AArch64::REV64v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
2712 }
2713 return 0;
2714}
2715
2716unsigned fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v2i32_r(unsigned Op0, bool Op0IsKill) {
2717 if ((!Subtarget->isLittleEndian())) {
2718 return fastEmitInst_r(AArch64::REV64v2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill);
2719 }
2720 return 0;
2721}
2722
2723unsigned fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v4f16_r(unsigned Op0, bool Op0IsKill) {
2724 if ((!Subtarget->isLittleEndian())) {
2725 return fastEmitInst_r(AArch64::REV64v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
2726 }
2727 return 0;
2728}
2729
2730unsigned fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v4bf16_r(unsigned Op0, bool Op0IsKill) {
2731 if ((!Subtarget->isLittleEndian())) {
2732 return fastEmitInst_r(AArch64::REV64v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
2733 }
2734 return 0;
2735}
2736
2737unsigned fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v2f32_r(unsigned Op0, bool Op0IsKill) {
2738 if ((!Subtarget->isLittleEndian())) {
2739 return fastEmitInst_r(AArch64::REV64v2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill);
2740 }
2741 return 0;
2742}
2743
2744unsigned fastEmit_ISD_BITCAST_MVT_v1f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2745switch (RetVT.SimpleTy) {
2746 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v8i8_r(Op0, Op0IsKill);
2747 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v4i16_r(Op0, Op0IsKill);
2748 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v2i32_r(Op0, Op0IsKill);
2749 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v4f16_r(Op0, Op0IsKill);
2750 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v4bf16_r(Op0, Op0IsKill);
2751 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v2f32_r(Op0, Op0IsKill);
2752 default: return 0;
2753}
2754}
2755
2756unsigned fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
2757 if ((!Subtarget->isLittleEndian())) {
2758 return fastEmitInst_r(AArch64::REV64v16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill);
2759 }
2760 return 0;
2761}
2762
2763unsigned fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
2764 if ((!Subtarget->isLittleEndian())) {
2765 return fastEmitInst_r(AArch64::REV64v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
2766 }
2767 return 0;
2768}
2769
2770unsigned fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
2771 if ((!Subtarget->isLittleEndian())) {
2772 return fastEmitInst_r(AArch64::REV64v4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill);
2773 }
2774 return 0;
2775}
2776
2777unsigned fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8f16_r(unsigned Op0, bool Op0IsKill) {
2778 if ((!Subtarget->isLittleEndian())) {
2779 return fastEmitInst_r(AArch64::REV64v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
2780 }
2781 return 0;
2782}
2783
2784unsigned fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8bf16_r(unsigned Op0, bool Op0IsKill) {
2785 if ((!Subtarget->isLittleEndian())) {
2786 return fastEmitInst_r(AArch64::REV64v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
2787 }
2788 return 0;
2789}
2790
2791unsigned fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v4f32_r(unsigned Op0, bool Op0IsKill) {
2792 if ((!Subtarget->isLittleEndian())) {
2793 return fastEmitInst_r(AArch64::REV64v4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill);
2794 }
2795 return 0;
2796}
2797
2798unsigned fastEmit_ISD_BITCAST_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2799switch (RetVT.SimpleTy) {
2800 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v16i8_r(Op0, Op0IsKill);
2801 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8i16_r(Op0, Op0IsKill);
2802 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v4i32_r(Op0, Op0IsKill);
2803 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8f16_r(Op0, Op0IsKill);
2804 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8bf16_r(Op0, Op0IsKill);
2805 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v4f32_r(Op0, Op0IsKill);
2806 default: return 0;
2807}
2808}
2809
2810unsigned fastEmit_ISD_BITCAST_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
2811 switch (VT.SimpleTy) {
2812 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_f64_r(RetVT, Op0, Op0IsKill);
2813 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v8i8_r(RetVT, Op0, Op0IsKill);
2814 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
2815 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v4i16_r(RetVT, Op0, Op0IsKill);
2816 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
2817 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v2i32_r(RetVT, Op0, Op0IsKill);
2818 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
2819 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v1i64_r(RetVT, Op0, Op0IsKill);
2820 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
2821 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v4f16_r(RetVT, Op0, Op0IsKill);
2822 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v8f16_r(RetVT, Op0, Op0IsKill);
2823 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v4bf16_r(RetVT, Op0, Op0IsKill);
2824 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v8bf16_r(RetVT, Op0, Op0IsKill);
2825 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v2f32_r(RetVT, Op0, Op0IsKill);
2826 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
2827 case MVT::v1f64: return fastEmit_ISD_BITCAST_MVT_v1f64_r(RetVT, Op0, Op0IsKill);
2828 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
2829 default: return 0;
2830 }
2831}
2832
2833// FastEmit functions for ISD::BITREVERSE.
2834
2835unsigned fastEmit_ISD_BITREVERSE_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2836 if (RetVT.SimpleTy != MVT::i32)
2837 return 0;
2838 return fastEmitInst_r(AArch64::RBITWr, &AArch64::GPR32RegClass, Op0, Op0IsKill);
2839}
2840
2841unsigned fastEmit_ISD_BITREVERSE_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2842 if (RetVT.SimpleTy != MVT::i64)
2843 return 0;
2844 return fastEmitInst_r(AArch64::RBITXr, &AArch64::GPR64RegClass, Op0, Op0IsKill);
2845}
2846
2847unsigned fastEmit_ISD_BITREVERSE_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
2848 switch (VT.SimpleTy) {
2849 case MVT::i32: return fastEmit_ISD_BITREVERSE_MVT_i32_r(RetVT, Op0, Op0IsKill);
2850 case MVT::i64: return fastEmit_ISD_BITREVERSE_MVT_i64_r(RetVT, Op0, Op0IsKill);
2851 default: return 0;
2852 }
2853}
2854
2855// FastEmit functions for ISD::BRIND.
2856
2857unsigned fastEmit_ISD_BRIND_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2858 if (RetVT.SimpleTy != MVT::isVoid)
2859 return 0;
2860 return fastEmitInst_r(AArch64::BR, &AArch64::GPR64RegClass, Op0, Op0IsKill);
2861}
2862
2863unsigned fastEmit_ISD_BRIND_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
2864 switch (VT.SimpleTy) {
2865 case MVT::i64: return fastEmit_ISD_BRIND_MVT_i64_r(RetVT, Op0, Op0IsKill);
2866 default: return 0;
2867 }
2868}
2869
2870// FastEmit functions for ISD::BSWAP.
2871
2872unsigned fastEmit_ISD_BSWAP_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2873 if (RetVT.SimpleTy != MVT::i32)
2874 return 0;
2875 return fastEmitInst_r(AArch64::REVWr, &AArch64::GPR32RegClass, Op0, Op0IsKill);
2876}
2877
2878unsigned fastEmit_ISD_BSWAP_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2879 if (RetVT.SimpleTy != MVT::i64)
2880 return 0;
2881 return fastEmitInst_r(AArch64::REVXr, &AArch64::GPR64RegClass, Op0, Op0IsKill);
2882}
2883
2884unsigned fastEmit_ISD_BSWAP_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
2885 switch (VT.SimpleTy) {
2886 case MVT::i32: return fastEmit_ISD_BSWAP_MVT_i32_r(RetVT, Op0, Op0IsKill);
2887 case MVT::i64: return fastEmit_ISD_BSWAP_MVT_i64_r(RetVT, Op0, Op0IsKill);
2888 default: return 0;
2889 }
2890}
2891
2892// FastEmit functions for ISD::CTLZ.
2893
2894unsigned fastEmit_ISD_CTLZ_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2895 if (RetVT.SimpleTy != MVT::i32)
2896 return 0;
2897 return fastEmitInst_r(AArch64::CLZWr, &AArch64::GPR32RegClass, Op0, Op0IsKill);
2898}
2899
2900unsigned fastEmit_ISD_CTLZ_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2901 if (RetVT.SimpleTy != MVT::i64)
2902 return 0;
2903 return fastEmitInst_r(AArch64::CLZXr, &AArch64::GPR64RegClass, Op0, Op0IsKill);
2904}
2905
2906unsigned fastEmit_ISD_CTLZ_MVT_v8i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2907 if (RetVT.SimpleTy != MVT::v8i8)
2908 return 0;
2909 if ((Subtarget->hasNEON())) {
2910 return fastEmitInst_r(AArch64::CLZv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill);
2911 }
2912 return 0;
2913}
2914
2915unsigned fastEmit_ISD_CTLZ_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2916 if (RetVT.SimpleTy != MVT::v16i8)
2917 return 0;
2918 if ((Subtarget->hasNEON())) {
2919 return fastEmitInst_r(AArch64::CLZv16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill);
2920 }
2921 return 0;
2922}
2923
2924unsigned fastEmit_ISD_CTLZ_MVT_v4i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2925 if (RetVT.SimpleTy != MVT::v4i16)
2926 return 0;
2927 if ((Subtarget->hasNEON())) {
2928 return fastEmitInst_r(AArch64::CLZv4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
2929 }
2930 return 0;
2931}
2932
2933unsigned fastEmit_ISD_CTLZ_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2934 if (RetVT.SimpleTy != MVT::v8i16)
2935 return 0;
2936 if ((Subtarget->hasNEON())) {
2937 return fastEmitInst_r(AArch64::CLZv8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
2938 }
2939 return 0;
2940}
2941
2942unsigned fastEmit_ISD_CTLZ_MVT_v2i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2943 if (RetVT.SimpleTy != MVT::v2i32)
2944 return 0;
2945 if ((Subtarget->hasNEON())) {
2946 return fastEmitInst_r(AArch64::CLZv2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill);
2947 }
2948 return 0;
2949}
2950
2951unsigned fastEmit_ISD_CTLZ_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2952 if (RetVT.SimpleTy != MVT::v4i32)
2953 return 0;
2954 if ((Subtarget->hasNEON())) {
2955 return fastEmitInst_r(AArch64::CLZv4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill);
2956 }
2957 return 0;
2958}
2959
2960unsigned fastEmit_ISD_CTLZ_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
2961 switch (VT.SimpleTy) {
2962 case MVT::i32: return fastEmit_ISD_CTLZ_MVT_i32_r(RetVT, Op0, Op0IsKill);
2963 case MVT::i64: return fastEmit_ISD_CTLZ_MVT_i64_r(RetVT, Op0, Op0IsKill);
2964 case MVT::v8i8: return fastEmit_ISD_CTLZ_MVT_v8i8_r(RetVT, Op0, Op0IsKill);
2965 case MVT::v16i8: return fastEmit_ISD_CTLZ_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
2966 case MVT::v4i16: return fastEmit_ISD_CTLZ_MVT_v4i16_r(RetVT, Op0, Op0IsKill);
2967 case MVT::v8i16: return fastEmit_ISD_CTLZ_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
2968 case MVT::v2i32: return fastEmit_ISD_CTLZ_MVT_v2i32_r(RetVT, Op0, Op0IsKill);
2969 case MVT::v4i32: return fastEmit_ISD_CTLZ_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
2970 default: return 0;
2971 }
2972}
2973
2974// FastEmit functions for ISD::CTPOP.
2975
2976unsigned fastEmit_ISD_CTPOP_MVT_v8i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2977 if (RetVT.SimpleTy != MVT::v8i8)
2978 return 0;
2979 if ((Subtarget->hasNEON())) {
2980 return fastEmitInst_r(AArch64::CNTv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill);
2981 }
2982 return 0;
2983}
2984
2985unsigned fastEmit_ISD_CTPOP_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2986 if (RetVT.SimpleTy != MVT::v16i8)
2987 return 0;
2988 if ((Subtarget->hasNEON())) {
2989 return fastEmitInst_r(AArch64::CNTv16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill);
2990 }
2991 return 0;
2992}
2993
2994unsigned fastEmit_ISD_CTPOP_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
2995 switch (VT.SimpleTy) {
2996 case MVT::v8i8: return fastEmit_ISD_CTPOP_MVT_v8i8_r(RetVT, Op0, Op0IsKill);
2997 case MVT::v16i8: return fastEmit_ISD_CTPOP_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
2998 default: return 0;
2999 }
3000}
3001
3002// FastEmit functions for ISD::FABS.
3003
3004unsigned fastEmit_ISD_FABS_MVT_f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3005 if (RetVT.SimpleTy != MVT::f16)
3006 return 0;
3007 if ((Subtarget->hasFullFP16())) {
3008 return fastEmitInst_r(AArch64::FABSHr, &AArch64::FPR16RegClass, Op0, Op0IsKill);
3009 }
3010 return 0;
3011}
3012
3013unsigned fastEmit_ISD_FABS_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3014 if (RetVT.SimpleTy != MVT::f32)
3015 return 0;
3016 if ((Subtarget->hasFPARMv8())) {
3017 return fastEmitInst_r(AArch64::FABSSr, &AArch64::FPR32RegClass, Op0, Op0IsKill);
3018 }
3019 return 0;
3020}
3021
3022unsigned fastEmit_ISD_FABS_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3023 if (RetVT.SimpleTy != MVT::f64)
3024 return 0;
3025 if ((Subtarget->hasFPARMv8())) {
3026 return fastEmitInst_r(AArch64::FABSDr, &AArch64::FPR64RegClass, Op0, Op0IsKill);
3027 }
3028 return 0;
3029}
3030
3031unsigned fastEmit_ISD_FABS_MVT_v4f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3032 if (RetVT.SimpleTy != MVT::v4f16)
3033 return 0;
3034 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
3035 return fastEmitInst_r(AArch64::FABSv4f16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
3036 }
3037 return 0;
3038}
3039
3040unsigned fastEmit_ISD_FABS_MVT_v8f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3041 if (RetVT.SimpleTy != MVT::v8f16)
3042 return 0;
3043 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
3044 return fastEmitInst_r(AArch64::FABSv8f16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
3045 }
3046 return 0;
3047}
3048
3049unsigned fastEmit_ISD_FABS_MVT_v2f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3050 if (RetVT.SimpleTy != MVT::v2f32)
3051 return 0;
3052 if ((Subtarget->hasNEON())) {
3053 return fastEmitInst_r(AArch64::FABSv2f32, &AArch64::FPR64RegClass, Op0, Op0IsKill);
3054 }
3055 return 0;
3056}
3057
3058unsigned fastEmit_ISD_FABS_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3059 if (RetVT.SimpleTy != MVT::v4f32)
3060 return 0;
3061 if ((Subtarget->hasNEON())) {
3062 return fastEmitInst_r(AArch64::FABSv4f32, &AArch64::FPR128RegClass, Op0, Op0IsKill);
3063 }
3064 return 0;
3065}
3066
3067unsigned fastEmit_ISD_FABS_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3068 if (RetVT.SimpleTy != MVT::v2f64)
3069 return 0;
3070 if ((Subtarget->hasNEON())) {
3071 return fastEmitInst_r(AArch64::FABSv2f64, &AArch64::FPR128RegClass, Op0, Op0IsKill);
3072 }
3073 return 0;
3074}
3075
3076unsigned fastEmit_ISD_FABS_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
3077 switch (VT.SimpleTy) {
3078 case MVT::f16: return fastEmit_ISD_FABS_MVT_f16_r(RetVT, Op0, Op0IsKill);
3079 case MVT::f32: return fastEmit_ISD_FABS_MVT_f32_r(RetVT, Op0, Op0IsKill);
3080 case MVT::f64: return fastEmit_ISD_FABS_MVT_f64_r(RetVT, Op0, Op0IsKill);
3081 case MVT::v4f16: return fastEmit_ISD_FABS_MVT_v4f16_r(RetVT, Op0, Op0IsKill);
3082 case MVT::v8f16: return fastEmit_ISD_FABS_MVT_v8f16_r(RetVT, Op0, Op0IsKill);
3083 case MVT::v2f32: return fastEmit_ISD_FABS_MVT_v2f32_r(RetVT, Op0, Op0IsKill);
3084 case MVT::v4f32: return fastEmit_ISD_FABS_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
3085 case MVT::v2f64: return fastEmit_ISD_FABS_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
3086 default: return 0;
3087 }
3088}
3089
3090// FastEmit functions for ISD::FCEIL.
3091
3092unsigned fastEmit_ISD_FCEIL_MVT_f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3093 if (RetVT.SimpleTy != MVT::f16)
3094 return 0;
3095 if ((Subtarget->hasFullFP16())) {
3096 return fastEmitInst_r(AArch64::FRINTPHr, &AArch64::FPR16RegClass, Op0, Op0IsKill);
3097 }
3098 return 0;
3099}
3100
3101unsigned fastEmit_ISD_FCEIL_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3102 if (RetVT.SimpleTy != MVT::f32)
3103 return 0;
3104 if ((Subtarget->hasFPARMv8())) {
3105 return fastEmitInst_r(AArch64::FRINTPSr, &AArch64::FPR32RegClass, Op0, Op0IsKill);
3106 }
3107 return 0;
3108}
3109
3110unsigned fastEmit_ISD_FCEIL_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3111 if (RetVT.SimpleTy != MVT::f64)
3112 return 0;
3113 if ((Subtarget->hasFPARMv8())) {
3114 return fastEmitInst_r(AArch64::FRINTPDr, &AArch64::FPR64RegClass, Op0, Op0IsKill);
3115 }
3116 return 0;
3117}
3118
3119unsigned fastEmit_ISD_FCEIL_MVT_v4f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3120 if (RetVT.SimpleTy != MVT::v4f16)
3121 return 0;
3122 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
3123 return fastEmitInst_r(AArch64::FRINTPv4f16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
3124 }
3125 return 0;
3126}
3127
3128unsigned fastEmit_ISD_FCEIL_MVT_v8f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3129 if (RetVT.SimpleTy != MVT::v8f16)
3130 return 0;
3131 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
3132 return fastEmitInst_r(AArch64::FRINTPv8f16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
3133 }
3134 return 0;
3135}
3136
3137unsigned fastEmit_ISD_FCEIL_MVT_v2f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3138 if (RetVT.SimpleTy != MVT::v2f32)
3139 return 0;
3140 if ((Subtarget->hasNEON())) {
3141 return fastEmitInst_r(AArch64::FRINTPv2f32, &AArch64::FPR64RegClass, Op0, Op0IsKill);
3142 }
3143 return 0;
3144}
3145
3146unsigned fastEmit_ISD_FCEIL_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3147 if (RetVT.SimpleTy != MVT::v4f32)
3148 return 0;
3149 if ((Subtarget->hasNEON())) {
3150 return fastEmitInst_r(AArch64::FRINTPv4f32, &AArch64::FPR128RegClass, Op0, Op0IsKill);
3151 }
3152 return 0;
3153}
3154
3155unsigned fastEmit_ISD_FCEIL_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3156 if (RetVT.SimpleTy != MVT::v2f64)
3157 return 0;
3158 if ((Subtarget->hasNEON())) {
3159 return fastEmitInst_r(AArch64::FRINTPv2f64, &AArch64::FPR128RegClass, Op0, Op0IsKill);
3160 }
3161 return 0;
3162}
3163
3164unsigned fastEmit_ISD_FCEIL_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
3165 switch (VT.SimpleTy) {
3166 case MVT::f16: return fastEmit_ISD_FCEIL_MVT_f16_r(RetVT, Op0, Op0IsKill);
3167 case MVT::f32: return fastEmit_ISD_FCEIL_MVT_f32_r(RetVT, Op0, Op0IsKill);
3168 case MVT::f64: return fastEmit_ISD_FCEIL_MVT_f64_r(RetVT, Op0, Op0IsKill);
3169 case MVT::v4f16: return fastEmit_ISD_FCEIL_MVT_v4f16_r(RetVT, Op0, Op0IsKill);
3170 case MVT::v8f16: return fastEmit_ISD_FCEIL_MVT_v8f16_r(RetVT, Op0, Op0IsKill);
3171 case MVT::v2f32: return fastEmit_ISD_FCEIL_MVT_v2f32_r(RetVT, Op0, Op0IsKill);
3172 case MVT::v4f32: return fastEmit_ISD_FCEIL_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
3173 case MVT::v2f64: return fastEmit_ISD_FCEIL_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
3174 default: return 0;
3175 }
3176}
3177
3178// FastEmit functions for ISD::FFLOOR.
3179
3180unsigned fastEmit_ISD_FFLOOR_MVT_f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3181 if (RetVT.SimpleTy != MVT::f16)
3182 return 0;
3183 if ((Subtarget->hasFullFP16())) {
3184 return fastEmitInst_r(AArch64::FRINTMHr, &AArch64::FPR16RegClass, Op0, Op0IsKill);
3185 }
3186 return 0;
3187}
3188
3189unsigned fastEmit_ISD_FFLOOR_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3190 if (RetVT.SimpleTy != MVT::f32)
3191 return 0;
3192 if ((Subtarget->hasFPARMv8())) {
3193 return fastEmitInst_r(AArch64::FRINTMSr, &AArch64::FPR32RegClass, Op0, Op0IsKill);
3194 }
3195 return 0;
3196}
3197
3198unsigned fastEmit_ISD_FFLOOR_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3199 if (RetVT.SimpleTy != MVT::f64)
3200 return 0;
3201 if ((Subtarget->hasFPARMv8())) {
3202 return fastEmitInst_r(AArch64::FRINTMDr, &AArch64::FPR64RegClass, Op0, Op0IsKill);
3203 }
3204 return 0;
3205}
3206
3207unsigned fastEmit_ISD_FFLOOR_MVT_v4f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3208 if (RetVT.SimpleTy != MVT::v4f16)
3209 return 0;
3210 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
3211 return fastEmitInst_r(AArch64::FRINTMv4f16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
3212 }
3213 return 0;
3214}
3215
3216unsigned fastEmit_ISD_FFLOOR_MVT_v8f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3217 if (RetVT.SimpleTy != MVT::v8f16)
3218 return 0;
3219 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
3220 return fastEmitInst_r(AArch64::FRINTMv8f16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
3221 }
3222 return 0;
3223}
3224
3225unsigned fastEmit_ISD_FFLOOR_MVT_v2f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3226 if (RetVT.SimpleTy != MVT::v2f32)
3227 return 0;
3228 if ((Subtarget->hasNEON())) {
3229 return fastEmitInst_r(AArch64::FRINTMv2f32, &AArch64::FPR64RegClass, Op0, Op0IsKill);
3230 }
3231 return 0;
3232}
3233
3234unsigned fastEmit_ISD_FFLOOR_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3235 if (RetVT.SimpleTy != MVT::v4f32)
3236 return 0;
3237 if ((Subtarget->hasNEON())) {
3238 return fastEmitInst_r(AArch64::FRINTMv4f32, &AArch64::FPR128RegClass, Op0, Op0IsKill);
3239 }
3240 return 0;
3241}
3242
3243unsigned fastEmit_ISD_FFLOOR_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3244 if (RetVT.SimpleTy != MVT::v2f64)
3245 return 0;
3246 if ((Subtarget->hasNEON())) {
3247 return fastEmitInst_r(AArch64::FRINTMv2f64, &AArch64::FPR128RegClass, Op0, Op0IsKill);
3248 }
3249 return 0;
3250}
3251
3252unsigned fastEmit_ISD_FFLOOR_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
3253 switch (VT.SimpleTy) {
3254 case MVT::f16: return fastEmit_ISD_FFLOOR_MVT_f16_r(RetVT, Op0, Op0IsKill);
3255 case MVT::f32: return fastEmit_ISD_FFLOOR_MVT_f32_r(RetVT, Op0, Op0IsKill);
3256 case MVT::f64: return fastEmit_ISD_FFLOOR_MVT_f64_r(RetVT, Op0, Op0IsKill);
3257 case MVT::v4f16: return fastEmit_ISD_FFLOOR_MVT_v4f16_r(RetVT, Op0, Op0IsKill);
3258 case MVT::v8f16: return fastEmit_ISD_FFLOOR_MVT_v8f16_r(RetVT, Op0, Op0IsKill);
3259 case MVT::v2f32: return fastEmit_ISD_FFLOOR_MVT_v2f32_r(RetVT, Op0, Op0IsKill);
3260 case MVT::v4f32: return fastEmit_ISD_FFLOOR_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
3261 case MVT::v2f64: return fastEmit_ISD_FFLOOR_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
3262 default: return 0;
3263 }
3264}
3265
3266// FastEmit functions for ISD::FNEARBYINT.
3267
3268unsigned fastEmit_ISD_FNEARBYINT_MVT_f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3269 if (RetVT.SimpleTy != MVT::f16)
3270 return 0;
3271 if ((Subtarget->hasFullFP16())) {
3272 return fastEmitInst_r(AArch64::FRINTIHr, &AArch64::FPR16RegClass, Op0, Op0IsKill);
3273 }
3274 return 0;
3275}
3276
3277unsigned fastEmit_ISD_FNEARBYINT_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3278 if (RetVT.SimpleTy != MVT::f32)
3279 return 0;
3280 if ((Subtarget->hasFPARMv8())) {
3281 return fastEmitInst_r(AArch64::FRINTISr, &AArch64::FPR32RegClass, Op0, Op0IsKill);
3282 }
3283 return 0;
3284}
3285
3286unsigned fastEmit_ISD_FNEARBYINT_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3287 if (RetVT.SimpleTy != MVT::f64)
3288 return 0;
3289 if ((Subtarget->hasFPARMv8())) {
3290 return fastEmitInst_r(AArch64::FRINTIDr, &AArch64::FPR64RegClass, Op0, Op0IsKill);
3291 }
3292 return 0;
3293}
3294
3295unsigned fastEmit_ISD_FNEARBYINT_MVT_v4f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3296 if (RetVT.SimpleTy != MVT::v4f16)
3297 return 0;
3298 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
3299 return fastEmitInst_r(AArch64::FRINTIv4f16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
3300 }
3301 return 0;
3302}
3303
3304unsigned fastEmit_ISD_FNEARBYINT_MVT_v8f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3305 if (RetVT.SimpleTy != MVT::v8f16)
3306 return 0;
3307 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
3308 return fastEmitInst_r(AArch64::FRINTIv8f16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
3309 }
3310 return 0;
3311}
3312
3313unsigned fastEmit_ISD_FNEARBYINT_MVT_v2f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3314 if (RetVT.SimpleTy != MVT::v2f32)
3315 return 0;
3316 if ((Subtarget->hasNEON())) {
3317 return fastEmitInst_r(AArch64::FRINTIv2f32, &AArch64::FPR64RegClass, Op0, Op0IsKill);
3318 }
3319 return 0;
3320}
3321
3322unsigned fastEmit_ISD_FNEARBYINT_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3323 if (RetVT.SimpleTy != MVT::v4f32)
3324 return 0;
3325 if ((Subtarget->hasNEON())) {
3326 return fastEmitInst_r(AArch64::FRINTIv4f32, &AArch64::FPR128RegClass, Op0, Op0IsKill);
3327 }
3328 return 0;
3329}
3330
3331unsigned fastEmit_ISD_FNEARBYINT_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3332 if (RetVT.SimpleTy != MVT::v2f64)
3333 return 0;
3334 if ((Subtarget->hasNEON())) {
3335 return fastEmitInst_r(AArch64::FRINTIv2f64, &AArch64::FPR128RegClass, Op0, Op0IsKill);
3336 }
3337 return 0;
3338}
3339
3340unsigned fastEmit_ISD_FNEARBYINT_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
3341 switch (VT.SimpleTy) {
3342 case MVT::f16: return fastEmit_ISD_FNEARBYINT_MVT_f16_r(RetVT, Op0, Op0IsKill);
3343 case MVT::f32: return fastEmit_ISD_FNEARBYINT_MVT_f32_r(RetVT, Op0, Op0IsKill);
3344 case MVT::f64: return fastEmit_ISD_FNEARBYINT_MVT_f64_r(RetVT, Op0, Op0IsKill);
3345 case MVT::v4f16: return fastEmit_ISD_FNEARBYINT_MVT_v4f16_r(RetVT, Op0, Op0IsKill);
3346 case MVT::v8f16: return fastEmit_ISD_FNEARBYINT_MVT_v8f16_r(RetVT, Op0, Op0IsKill);
3347 case MVT::v2f32: return fastEmit_ISD_FNEARBYINT_MVT_v2f32_r(RetVT, Op0, Op0IsKill);
3348 case MVT::v4f32: return fastEmit_ISD_FNEARBYINT_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
3349 case MVT::v2f64: return fastEmit_ISD_FNEARBYINT_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
3350 default: return 0;
3351 }
3352}
3353
3354// FastEmit functions for ISD::FNEG.
3355
3356unsigned fastEmit_ISD_FNEG_MVT_f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3357 if (RetVT.SimpleTy != MVT::f16)
3358 return 0;
3359 if ((Subtarget->hasFullFP16())) {
3360 return fastEmitInst_r(AArch64::FNEGHr, &AArch64::FPR16RegClass, Op0, Op0IsKill);
3361 }
3362 return 0;
3363}
3364
3365unsigned fastEmit_ISD_FNEG_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3366 if (RetVT.SimpleTy != MVT::f32)
3367 return 0;
3368 if ((Subtarget->hasFPARMv8())) {
3369 return fastEmitInst_r(AArch64::FNEGSr, &AArch64::FPR32RegClass, Op0, Op0IsKill);
3370 }
3371 return 0;
3372}
3373
3374unsigned fastEmit_ISD_FNEG_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3375 if (RetVT.SimpleTy != MVT::f64)
3376 return 0;
3377 if ((Subtarget->hasFPARMv8())) {
3378 return fastEmitInst_r(AArch64::FNEGDr, &AArch64::FPR64RegClass, Op0, Op0IsKill);
3379 }
3380 return 0;
3381}
3382
3383unsigned fastEmit_ISD_FNEG_MVT_v4f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3384 if (RetVT.SimpleTy != MVT::v4f16)
3385 return 0;
3386 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
3387 return fastEmitInst_r(AArch64::FNEGv4f16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
3388 }
3389 return 0;
3390}
3391
3392unsigned fastEmit_ISD_FNEG_MVT_v8f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3393 if (RetVT.SimpleTy != MVT::v8f16)
3394 return 0;
3395 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
3396 return fastEmitInst_r(AArch64::FNEGv8f16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
3397 }
3398 return 0;
3399}
3400
3401unsigned fastEmit_ISD_FNEG_MVT_v2f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3402 if (RetVT.SimpleTy != MVT::v2f32)
3403 return 0;
3404 if ((Subtarget->hasNEON())) {
3405 return fastEmitInst_r(AArch64::FNEGv2f32, &AArch64::FPR64RegClass, Op0, Op0IsKill);
3406 }
3407 return 0;
3408}
3409
3410unsigned fastEmit_ISD_FNEG_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3411 if (RetVT.SimpleTy != MVT::v4f32)
3412 return 0;
3413 if ((Subtarget->hasNEON())) {
3414 return fastEmitInst_r(AArch64::FNEGv4f32, &AArch64::FPR128RegClass, Op0, Op0IsKill);
3415 }
3416 return 0;
3417}
3418
3419unsigned fastEmit_ISD_FNEG_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3420 if (RetVT.SimpleTy != MVT::v2f64)
3421 return 0;
3422 if ((Subtarget->hasNEON())) {
3423 return fastEmitInst_r(AArch64::FNEGv2f64, &AArch64::FPR128RegClass, Op0, Op0IsKill);
3424 }
3425 return 0;
3426}
3427
3428unsigned fastEmit_ISD_FNEG_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
3429 switch (VT.SimpleTy) {
3430 case MVT::f16: return fastEmit_ISD_FNEG_MVT_f16_r(RetVT, Op0, Op0IsKill);
3431 case MVT::f32: return fastEmit_ISD_FNEG_MVT_f32_r(RetVT, Op0, Op0IsKill);
3432 case MVT::f64: return fastEmit_ISD_FNEG_MVT_f64_r(RetVT, Op0, Op0IsKill);
3433 case MVT::v4f16: return fastEmit_ISD_FNEG_MVT_v4f16_r(RetVT, Op0, Op0IsKill);
3434 case MVT::v8f16: return fastEmit_ISD_FNEG_MVT_v8f16_r(RetVT, Op0, Op0IsKill);
3435 case MVT::v2f32: return fastEmit_ISD_FNEG_MVT_v2f32_r(RetVT, Op0, Op0IsKill);
3436 case MVT::v4f32: return fastEmit_ISD_FNEG_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
3437 case MVT::v2f64: return fastEmit_ISD_FNEG_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
3438 default: return 0;
3439 }
3440}
3441
3442// FastEmit functions for ISD::FP_EXTEND.
3443
3444unsigned fastEmit_ISD_FP_EXTEND_MVT_f16_MVT_f32_r(unsigned Op0, bool Op0IsKill) {
3445 if ((Subtarget->hasFPARMv8())) {
3446 return fastEmitInst_r(AArch64::FCVTSHr, &AArch64::FPR32RegClass, Op0, Op0IsKill);
3447 }
3448 return 0;
3449}
3450
3451unsigned fastEmit_ISD_FP_EXTEND_MVT_f16_MVT_f64_r(unsigned Op0, bool Op0IsKill) {
3452 if ((Subtarget->hasFPARMv8())) {
3453 return fastEmitInst_r(AArch64::FCVTDHr, &AArch64::FPR64RegClass, Op0, Op0IsKill);
3454 }
3455 return 0;
3456}
3457
3458unsigned fastEmit_ISD_FP_EXTEND_MVT_f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3459switch (RetVT.SimpleTy) {
3460 case MVT::f32: return fastEmit_ISD_FP_EXTEND_MVT_f16_MVT_f32_r(Op0, Op0IsKill);
3461 case MVT::f64: return fastEmit_ISD_FP_EXTEND_MVT_f16_MVT_f64_r(Op0, Op0IsKill);
3462 default: return 0;
3463}
3464}
3465
3466unsigned fastEmit_ISD_FP_EXTEND_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3467 if (RetVT.SimpleTy != MVT::f64)
3468 return 0;
3469 if ((Subtarget->hasFPARMv8())) {
3470 return fastEmitInst_r(AArch64::FCVTDSr, &AArch64::FPR64RegClass, Op0, Op0IsKill);
3471 }
3472 return 0;
3473}
3474
3475unsigned fastEmit_ISD_FP_EXTEND_MVT_v4f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3476 if (RetVT.SimpleTy != MVT::v4f32)
3477 return 0;
3478 return fastEmitInst_r(AArch64::FCVTLv4i16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
3479}
3480
3481unsigned fastEmit_ISD_FP_EXTEND_MVT_v2f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3482 if (RetVT.SimpleTy != MVT::v2f64)
3483 return 0;
3484 return fastEmitInst_r(AArch64::FCVTLv2i32, &AArch64::FPR128RegClass, Op0, Op0IsKill);
3485}
3486
3487unsigned fastEmit_ISD_FP_EXTEND_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
3488 switch (VT.SimpleTy) {
3489 case MVT::f16: return fastEmit_ISD_FP_EXTEND_MVT_f16_r(RetVT, Op0, Op0IsKill);
3490 case MVT::f32: return fastEmit_ISD_FP_EXTEND_MVT_f32_r(RetVT, Op0, Op0IsKill);
3491 case MVT::v4f16: return fastEmit_ISD_FP_EXTEND_MVT_v4f16_r(RetVT, Op0, Op0IsKill);
3492 case MVT::v2f32: return fastEmit_ISD_FP_EXTEND_MVT_v2f32_r(RetVT, Op0, Op0IsKill);
3493 default: return 0;
3494 }
3495}
3496
3497// FastEmit functions for ISD::FP_ROUND.
3498
3499unsigned fastEmit_ISD_FP_ROUND_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3500 if (RetVT.SimpleTy != MVT::f16)
3501 return 0;
3502 if ((Subtarget->hasFPARMv8())) {
3503 return fastEmitInst_r(AArch64::FCVTHSr, &AArch64::FPR16RegClass, Op0, Op0IsKill);
3504 }
3505 return 0;
3506}
3507
3508unsigned fastEmit_ISD_FP_ROUND_MVT_f64_MVT_f16_r(unsigned Op0, bool Op0IsKill) {
3509 if ((Subtarget->hasFPARMv8())) {
3510 return fastEmitInst_r(AArch64::FCVTHDr, &AArch64::FPR16RegClass, Op0, Op0IsKill);
3511 }
3512 return 0;
3513}
3514
3515unsigned fastEmit_ISD_FP_ROUND_MVT_f64_MVT_f32_r(unsigned Op0, bool Op0IsKill) {
3516 if ((Subtarget->hasFPARMv8())) {
3517 return fastEmitInst_r(AArch64::FCVTSDr, &AArch64::FPR32RegClass, Op0, Op0IsKill);
3518 }
3519 return 0;
3520}
3521
3522unsigned fastEmit_ISD_FP_ROUND_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3523switch (RetVT.SimpleTy) {
3524 case MVT::f16: return fastEmit_ISD_FP_ROUND_MVT_f64_MVT_f16_r(Op0, Op0IsKill);
3525 case MVT::f32: return fastEmit_ISD_FP_ROUND_MVT_f64_MVT_f32_r(Op0, Op0IsKill);
3526 default: return 0;
3527}
3528}
3529
3530unsigned fastEmit_ISD_FP_ROUND_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3531 if (RetVT.SimpleTy != MVT::v4f16)
3532 return 0;
3533 return fastEmitInst_r(AArch64::FCVTNv4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
3534}
3535
3536unsigned fastEmit_ISD_FP_ROUND_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3537 if (RetVT.SimpleTy != MVT::v2f32)
3538 return 0;
3539 return fastEmitInst_r(AArch64::FCVTNv2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill);
3540}
3541
3542unsigned fastEmit_ISD_FP_ROUND_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
3543 switch (VT.SimpleTy) {
3544 case MVT::f32: return fastEmit_ISD_FP_ROUND_MVT_f32_r(RetVT, Op0, Op0IsKill);
3545 case MVT::f64: return fastEmit_ISD_FP_ROUND_MVT_f64_r(RetVT, Op0, Op0IsKill);
3546 case MVT::v4f32: return fastEmit_ISD_FP_ROUND_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
3547 case MVT::v2f64: return fastEmit_ISD_FP_ROUND_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
3548 default: return 0;
3549 }
3550}
3551
3552// FastEmit functions for ISD::FP_TO_SINT.
3553
3554unsigned fastEmit_ISD_FP_TO_SINT_MVT_f16_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
3555 if ((Subtarget->hasFullFP16())) {
3556 return fastEmitInst_r(AArch64::FCVTZSUWHr, &AArch64::GPR32RegClass, Op0, Op0IsKill);
3557 }
3558 return 0;
3559}
3560
3561unsigned fastEmit_ISD_FP_TO_SINT_MVT_f16_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
3562 if ((Subtarget->hasFullFP16())) {
3563 return fastEmitInst_r(AArch64::FCVTZSUXHr, &AArch64::GPR64RegClass, Op0, Op0IsKill);
3564 }
3565 return 0;
3566}
3567
3568unsigned fastEmit_ISD_FP_TO_SINT_MVT_f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3569switch (RetVT.SimpleTy) {
3570 case MVT::i32: return fastEmit_ISD_FP_TO_SINT_MVT_f16_MVT_i32_r(Op0, Op0IsKill);
3571 case MVT::i64: return fastEmit_ISD_FP_TO_SINT_MVT_f16_MVT_i64_r(Op0, Op0IsKill);
3572 default: return 0;
3573}
3574}
3575
3576unsigned fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
3577 if ((Subtarget->hasFPARMv8())) {
3578 return fastEmitInst_r(AArch64::FCVTZSUWSr, &AArch64::GPR32RegClass, Op0, Op0IsKill);
3579 }
3580 return 0;
3581}
3582
3583unsigned fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
3584 if ((Subtarget->hasFPARMv8())) {
3585 return fastEmitInst_r(AArch64::FCVTZSUXSr, &AArch64::GPR64RegClass, Op0, Op0IsKill);
3586 }
3587 return 0;
3588}
3589
3590unsigned fastEmit_ISD_FP_TO_SINT_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3591switch (RetVT.SimpleTy) {
3592 case MVT::i32: return fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i32_r(Op0, Op0IsKill);
3593 case MVT::i64: return fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i64_r(Op0, Op0IsKill);
3594 default: return 0;
3595}
3596}
3597
3598unsigned fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
3599 if ((Subtarget->hasFPARMv8())) {
3600 return fastEmitInst_r(AArch64::FCVTZSUWDr, &AArch64::GPR32RegClass, Op0, Op0IsKill);
3601 }
3602 return 0;
3603}
3604
3605unsigned fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
3606 if ((Subtarget->hasFPARMv8())) {
3607 return fastEmitInst_r(AArch64::FCVTZSUXDr, &AArch64::GPR64RegClass, Op0, Op0IsKill);
3608 }
3609 return 0;
3610}
3611
3612unsigned fastEmit_ISD_FP_TO_SINT_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3613switch (RetVT.SimpleTy) {
3614 case MVT::i32: return fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i32_r(Op0, Op0IsKill);
3615 case MVT::i64: return fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i64_r(Op0, Op0IsKill);
3616 default: return 0;
3617}
3618}
3619
3620unsigned fastEmit_ISD_FP_TO_SINT_MVT_v4f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3621 if (RetVT.SimpleTy != MVT::v4i16)
3622 return 0;
3623 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
3624 return fastEmitInst_r(AArch64::FCVTZSv4f16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
3625 }
3626 return 0;
3627}
3628
3629unsigned fastEmit_ISD_FP_TO_SINT_MVT_v8f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3630 if (RetVT.SimpleTy != MVT::v8i16)
3631 return 0;
3632 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
3633 return fastEmitInst_r(AArch64::FCVTZSv8f16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
3634 }
3635 return 0;
3636}
3637
3638unsigned fastEmit_ISD_FP_TO_SINT_MVT_v2f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3639 if (RetVT.SimpleTy != MVT::v2i32)
3640 return 0;
3641 if ((Subtarget->hasNEON())) {
3642 return fastEmitInst_r(AArch64::FCVTZSv2f32, &AArch64::FPR64RegClass, Op0, Op0IsKill);
3643 }
3644 return 0;
3645}
3646
3647unsigned fastEmit_ISD_FP_TO_SINT_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3648 if (RetVT.SimpleTy != MVT::v4i32)
3649 return 0;
3650 if ((Subtarget->hasNEON())) {
3651 return fastEmitInst_r(AArch64::FCVTZSv4f32, &AArch64::FPR128RegClass, Op0, Op0IsKill);
3652 }
3653 return 0;
3654}
3655
3656unsigned fastEmit_ISD_FP_TO_SINT_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3657 if (RetVT.SimpleTy != MVT::v2i64)
3658 return 0;
3659 if ((Subtarget->hasNEON())) {
3660 return fastEmitInst_r(AArch64::FCVTZSv2f64, &AArch64::FPR128RegClass, Op0, Op0IsKill);
3661 }
3662 return 0;
3663}
3664
3665unsigned fastEmit_ISD_FP_TO_SINT_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
3666 switch (VT.SimpleTy) {
3667 case MVT::f16: return fastEmit_ISD_FP_TO_SINT_MVT_f16_r(RetVT, Op0, Op0IsKill);
3668 case MVT::f32: return fastEmit_ISD_FP_TO_SINT_MVT_f32_r(RetVT, Op0, Op0IsKill);
3669 case MVT::f64: return fastEmit_ISD_FP_TO_SINT_MVT_f64_r(RetVT, Op0, Op0IsKill);
3670 case MVT::v4f16: return fastEmit_ISD_FP_TO_SINT_MVT_v4f16_r(RetVT, Op0, Op0IsKill);
3671 case MVT::v8f16: return fastEmit_ISD_FP_TO_SINT_MVT_v8f16_r(RetVT, Op0, Op0IsKill);
3672 case MVT::v2f32: return fastEmit_ISD_FP_TO_SINT_MVT_v2f32_r(RetVT, Op0, Op0IsKill);
3673 case MVT::v4f32: return fastEmit_ISD_FP_TO_SINT_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
3674 case MVT::v2f64: return fastEmit_ISD_FP_TO_SINT_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
3675 default: return 0;
3676 }
3677}
3678
3679// FastEmit functions for ISD::FP_TO_UINT.
3680
3681unsigned fastEmit_ISD_FP_TO_UINT_MVT_f16_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
3682 if ((Subtarget->hasFullFP16())) {
3683 return fastEmitInst_r(AArch64::FCVTZUUWHr, &AArch64::GPR32RegClass, Op0, Op0IsKill);
3684 }
3685 return 0;
3686}
3687
3688unsigned fastEmit_ISD_FP_TO_UINT_MVT_f16_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
3689 if ((Subtarget->hasFullFP16())) {
3690 return fastEmitInst_r(AArch64::FCVTZUUXHr, &AArch64::GPR64RegClass, Op0, Op0IsKill);
3691 }
3692 return 0;
3693}
3694
3695unsigned fastEmit_ISD_FP_TO_UINT_MVT_f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3696switch (RetVT.SimpleTy) {
3697 case MVT::i32: return fastEmit_ISD_FP_TO_UINT_MVT_f16_MVT_i32_r(Op0, Op0IsKill);
3698 case MVT::i64: return fastEmit_ISD_FP_TO_UINT_MVT_f16_MVT_i64_r(Op0, Op0IsKill);
3699 default: return 0;
3700}
3701}
3702
3703unsigned fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
3704 if ((Subtarget->hasFPARMv8())) {
3705 return fastEmitInst_r(AArch64::FCVTZUUWSr, &AArch64::GPR32RegClass, Op0, Op0IsKill);
3706 }
3707 return 0;
3708}
3709
3710unsigned fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
3711 if ((Subtarget->hasFPARMv8())) {
3712 return fastEmitInst_r(AArch64::FCVTZUUXSr, &AArch64::GPR64RegClass, Op0, Op0IsKill);
3713 }
3714 return 0;
3715}
3716
3717unsigned fastEmit_ISD_FP_TO_UINT_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3718switch (RetVT.SimpleTy) {
3719 case MVT::i32: return fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i32_r(Op0, Op0IsKill);
3720 case MVT::i64: return fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i64_r(Op0, Op0IsKill);
3721 default: return 0;
3722}
3723}
3724
3725unsigned fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
3726 if ((Subtarget->hasFPARMv8())) {
3727 return fastEmitInst_r(AArch64::FCVTZUUWDr, &AArch64::GPR32RegClass, Op0, Op0IsKill);
3728 }
3729 return 0;
3730}
3731
3732unsigned fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
3733 if ((Subtarget->hasFPARMv8())) {
3734 return fastEmitInst_r(AArch64::FCVTZUUXDr, &AArch64::GPR64RegClass, Op0, Op0IsKill);
3735 }
3736 return 0;
3737}
3738
3739unsigned fastEmit_ISD_FP_TO_UINT_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3740switch (RetVT.SimpleTy) {
3741 case MVT::i32: return fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i32_r(Op0, Op0IsKill);
3742 case MVT::i64: return fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i64_r(Op0, Op0IsKill);
3743 default: return 0;
3744}
3745}
3746
3747unsigned fastEmit_ISD_FP_TO_UINT_MVT_v4f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3748 if (RetVT.SimpleTy != MVT::v4i16)
3749 return 0;
3750 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
3751 return fastEmitInst_r(AArch64::FCVTZUv4f16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
3752 }
3753 return 0;
3754}
3755
3756unsigned fastEmit_ISD_FP_TO_UINT_MVT_v8f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3757 if (RetVT.SimpleTy != MVT::v8i16)
3758 return 0;
3759 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
3760 return fastEmitInst_r(AArch64::FCVTZUv8f16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
3761 }
3762 return 0;
3763}
3764
3765unsigned fastEmit_ISD_FP_TO_UINT_MVT_v2f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3766 if (RetVT.SimpleTy != MVT::v2i32)
3767 return 0;
3768 if ((Subtarget->hasNEON())) {
3769 return fastEmitInst_r(AArch64::FCVTZUv2f32, &AArch64::FPR64RegClass, Op0, Op0IsKill);
3770 }
3771 return 0;
3772}
3773
3774unsigned fastEmit_ISD_FP_TO_UINT_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3775 if (RetVT.SimpleTy != MVT::v4i32)
3776 return 0;
3777 if ((Subtarget->hasNEON())) {
3778 return fastEmitInst_r(AArch64::FCVTZUv4f32, &AArch64::FPR128RegClass, Op0, Op0IsKill);
3779 }
3780 return 0;
3781}
3782
3783unsigned fastEmit_ISD_FP_TO_UINT_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3784 if (RetVT.SimpleTy != MVT::v2i64)
3785 return 0;
3786 if ((Subtarget->hasNEON())) {
3787 return fastEmitInst_r(AArch64::FCVTZUv2f64, &AArch64::FPR128RegClass, Op0, Op0IsKill);
3788 }
3789 return 0;
3790}
3791
3792unsigned fastEmit_ISD_FP_TO_UINT_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
3793 switch (VT.SimpleTy) {
3794 case MVT::f16: return fastEmit_ISD_FP_TO_UINT_MVT_f16_r(RetVT, Op0, Op0IsKill);
3795 case MVT::f32: return fastEmit_ISD_FP_TO_UINT_MVT_f32_r(RetVT, Op0, Op0IsKill);
3796 case MVT::f64: return fastEmit_ISD_FP_TO_UINT_MVT_f64_r(RetVT, Op0, Op0IsKill);
3797 case MVT::v4f16: return fastEmit_ISD_FP_TO_UINT_MVT_v4f16_r(RetVT, Op0, Op0IsKill);
3798 case MVT::v8f16: return fastEmit_ISD_FP_TO_UINT_MVT_v8f16_r(RetVT, Op0, Op0IsKill);
3799 case MVT::v2f32: return fastEmit_ISD_FP_TO_UINT_MVT_v2f32_r(RetVT, Op0, Op0IsKill);
3800 case MVT::v4f32: return fastEmit_ISD_FP_TO_UINT_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
3801 case MVT::v2f64: return fastEmit_ISD_FP_TO_UINT_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
3802 default: return 0;
3803 }
3804}
3805
3806// FastEmit functions for ISD::FRINT.
3807
3808unsigned fastEmit_ISD_FRINT_MVT_f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3809 if (RetVT.SimpleTy != MVT::f16)
3810 return 0;
3811 if ((Subtarget->hasFullFP16())) {
3812 return fastEmitInst_r(AArch64::FRINTXHr, &AArch64::FPR16RegClass, Op0, Op0IsKill);
3813 }
3814 return 0;
3815}
3816
3817unsigned fastEmit_ISD_FRINT_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3818 if (RetVT.SimpleTy != MVT::f32)
3819 return 0;
3820 if ((Subtarget->hasFPARMv8())) {
3821 return fastEmitInst_r(AArch64::FRINTXSr, &AArch64::FPR32RegClass, Op0, Op0IsKill);
3822 }
3823 return 0;
3824}
3825
3826unsigned fastEmit_ISD_FRINT_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3827 if (RetVT.SimpleTy != MVT::f64)
3828 return 0;
3829 if ((Subtarget->hasFPARMv8())) {
3830 return fastEmitInst_r(AArch64::FRINTXDr, &AArch64::FPR64RegClass, Op0, Op0IsKill);
3831 }
3832 return 0;
3833}
3834
3835unsigned fastEmit_ISD_FRINT_MVT_v4f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3836 if (RetVT.SimpleTy != MVT::v4f16)
3837 return 0;
3838 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
3839 return fastEmitInst_r(AArch64::FRINTXv4f16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
3840 }
3841 return 0;
3842}
3843
3844unsigned fastEmit_ISD_FRINT_MVT_v8f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3845 if (RetVT.SimpleTy != MVT::v8f16)
3846 return 0;
3847 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
3848 return fastEmitInst_r(AArch64::FRINTXv8f16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
3849 }
3850 return 0;
3851}
3852
3853unsigned fastEmit_ISD_FRINT_MVT_v2f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3854 if (RetVT.SimpleTy != MVT::v2f32)
3855 return 0;
3856 if ((Subtarget->hasNEON())) {
3857 return fastEmitInst_r(AArch64::FRINTXv2f32, &AArch64::FPR64RegClass, Op0, Op0IsKill);
3858 }
3859 return 0;
3860}
3861
3862unsigned fastEmit_ISD_FRINT_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3863 if (RetVT.SimpleTy != MVT::v4f32)
3864 return 0;
3865 if ((Subtarget->hasNEON())) {
3866 return fastEmitInst_r(AArch64::FRINTXv4f32, &AArch64::FPR128RegClass, Op0, Op0IsKill);
3867 }
3868 return 0;
3869}
3870
3871unsigned fastEmit_ISD_FRINT_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3872 if (RetVT.SimpleTy != MVT::v2f64)
3873 return 0;
3874 if ((Subtarget->hasNEON())) {
3875 return fastEmitInst_r(AArch64::FRINTXv2f64, &AArch64::FPR128RegClass, Op0, Op0IsKill);
3876 }
3877 return 0;
3878}
3879
3880unsigned fastEmit_ISD_FRINT_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
3881 switch (VT.SimpleTy) {
3882 case MVT::f16: return fastEmit_ISD_FRINT_MVT_f16_r(RetVT, Op0, Op0IsKill);
3883 case MVT::f32: return fastEmit_ISD_FRINT_MVT_f32_r(RetVT, Op0, Op0IsKill);
3884 case MVT::f64: return fastEmit_ISD_FRINT_MVT_f64_r(RetVT, Op0, Op0IsKill);
3885 case MVT::v4f16: return fastEmit_ISD_FRINT_MVT_v4f16_r(RetVT, Op0, Op0IsKill);
3886 case MVT::v8f16: return fastEmit_ISD_FRINT_MVT_v8f16_r(RetVT, Op0, Op0IsKill);
3887 case MVT::v2f32: return fastEmit_ISD_FRINT_MVT_v2f32_r(RetVT, Op0, Op0IsKill);
3888 case MVT::v4f32: return fastEmit_ISD_FRINT_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
3889 case MVT::v2f64: return fastEmit_ISD_FRINT_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
3890 default: return 0;
3891 }
3892}
3893
3894// FastEmit functions for ISD::FROUND.
3895
3896unsigned fastEmit_ISD_FROUND_MVT_f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3897 if (RetVT.SimpleTy != MVT::f16)
3898 return 0;
3899 if ((Subtarget->hasFullFP16())) {
3900 return fastEmitInst_r(AArch64::FRINTAHr, &AArch64::FPR16RegClass, Op0, Op0IsKill);
3901 }
3902 return 0;
3903}
3904
3905unsigned fastEmit_ISD_FROUND_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3906 if (RetVT.SimpleTy != MVT::f32)
3907 return 0;
3908 if ((Subtarget->hasFPARMv8())) {
3909 return fastEmitInst_r(AArch64::FRINTASr, &AArch64::FPR32RegClass, Op0, Op0IsKill);
3910 }
3911 return 0;
3912}
3913
3914unsigned fastEmit_ISD_FROUND_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3915 if (RetVT.SimpleTy != MVT::f64)
3916 return 0;
3917 if ((Subtarget->hasFPARMv8())) {
3918 return fastEmitInst_r(AArch64::FRINTADr, &AArch64::FPR64RegClass, Op0, Op0IsKill);
3919 }
3920 return 0;
3921}
3922
3923unsigned fastEmit_ISD_FROUND_MVT_v4f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3924 if (RetVT.SimpleTy != MVT::v4f16)
3925 return 0;
3926 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
3927 return fastEmitInst_r(AArch64::FRINTAv4f16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
3928 }
3929 return 0;
3930}
3931
3932unsigned fastEmit_ISD_FROUND_MVT_v8f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3933 if (RetVT.SimpleTy != MVT::v8f16)
3934 return 0;
3935 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
3936 return fastEmitInst_r(AArch64::FRINTAv8f16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
3937 }
3938 return 0;
3939}
3940
3941unsigned fastEmit_ISD_FROUND_MVT_v2f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3942 if (RetVT.SimpleTy != MVT::v2f32)
3943 return 0;
3944 if ((Subtarget->hasNEON())) {
3945 return fastEmitInst_r(AArch64::FRINTAv2f32, &AArch64::FPR64RegClass, Op0, Op0IsKill);
3946 }
3947 return 0;
3948}
3949
3950unsigned fastEmit_ISD_FROUND_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3951 if (RetVT.SimpleTy != MVT::v4f32)
3952 return 0;
3953 if ((Subtarget->hasNEON())) {
3954 return fastEmitInst_r(AArch64::FRINTAv4f32, &AArch64::FPR128RegClass, Op0, Op0IsKill);
3955 }
3956 return 0;
3957}
3958
3959unsigned fastEmit_ISD_FROUND_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3960 if (RetVT.SimpleTy != MVT::v2f64)
3961 return 0;
3962 if ((Subtarget->hasNEON())) {
3963 return fastEmitInst_r(AArch64::FRINTAv2f64, &AArch64::FPR128RegClass, Op0, Op0IsKill);
3964 }
3965 return 0;
3966}
3967
3968unsigned fastEmit_ISD_FROUND_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
3969 switch (VT.SimpleTy) {
3970 case MVT::f16: return fastEmit_ISD_FROUND_MVT_f16_r(RetVT, Op0, Op0IsKill);
3971 case MVT::f32: return fastEmit_ISD_FROUND_MVT_f32_r(RetVT, Op0, Op0IsKill);
3972 case MVT::f64: return fastEmit_ISD_FROUND_MVT_f64_r(RetVT, Op0, Op0IsKill);
3973 case MVT::v4f16: return fastEmit_ISD_FROUND_MVT_v4f16_r(RetVT, Op0, Op0IsKill);
3974 case MVT::v8f16: return fastEmit_ISD_FROUND_MVT_v8f16_r(RetVT, Op0, Op0IsKill);
3975 case MVT::v2f32: return fastEmit_ISD_FROUND_MVT_v2f32_r(RetVT, Op0, Op0IsKill);
3976 case MVT::v4f32: return fastEmit_ISD_FROUND_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
3977 case MVT::v2f64: return fastEmit_ISD_FROUND_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
3978 default: return 0;
3979 }
3980}
3981
3982// FastEmit functions for ISD::FSQRT.
3983
3984unsigned fastEmit_ISD_FSQRT_MVT_f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3985 if (RetVT.SimpleTy != MVT::f16)
3986 return 0;
3987 if ((Subtarget->hasFullFP16())) {
3988 return fastEmitInst_r(AArch64::FSQRTHr, &AArch64::FPR16RegClass, Op0, Op0IsKill);
3989 }
3990 return 0;
3991}
3992
3993unsigned fastEmit_ISD_FSQRT_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3994 if (RetVT.SimpleTy != MVT::f32)
3995 return 0;
3996 if ((Subtarget->hasFPARMv8())) {
3997 return fastEmitInst_r(AArch64::FSQRTSr, &AArch64::FPR32RegClass, Op0, Op0IsKill);
3998 }
3999 return 0;
4000}
4001
4002unsigned fastEmit_ISD_FSQRT_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4003 if (RetVT.SimpleTy != MVT::f64)
4004 return 0;
4005 if ((Subtarget->hasFPARMv8())) {
4006 return fastEmitInst_r(AArch64::FSQRTDr, &AArch64::FPR64RegClass, Op0, Op0IsKill);
4007 }
4008 return 0;
4009}
4010
4011unsigned fastEmit_ISD_FSQRT_MVT_v4f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4012 if (RetVT.SimpleTy != MVT::v4f16)
4013 return 0;
4014 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
4015 return fastEmitInst_r(AArch64::FSQRTv4f16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
4016 }
4017 return 0;
4018}
4019
4020unsigned fastEmit_ISD_FSQRT_MVT_v8f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4021 if (RetVT.SimpleTy != MVT::v8f16)
4022 return 0;
4023 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
4024 return fastEmitInst_r(AArch64::FSQRTv8f16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
4025 }
4026 return 0;
4027}
4028
4029unsigned fastEmit_ISD_FSQRT_MVT_v2f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4030 if (RetVT.SimpleTy != MVT::v2f32)
4031 return 0;
4032 if ((Subtarget->hasNEON())) {
4033 return fastEmitInst_r(AArch64::FSQRTv2f32, &AArch64::FPR64RegClass, Op0, Op0IsKill);
4034 }
4035 return 0;
4036}
4037
4038unsigned fastEmit_ISD_FSQRT_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4039 if (RetVT.SimpleTy != MVT::v4f32)
4040 return 0;
4041 if ((Subtarget->hasNEON())) {
4042 return fastEmitInst_r(AArch64::FSQRTv4f32, &AArch64::FPR128RegClass, Op0, Op0IsKill);
4043 }
4044 return 0;
4045}
4046
4047unsigned fastEmit_ISD_FSQRT_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4048 if (RetVT.SimpleTy != MVT::v2f64)
4049 return 0;
4050 if ((Subtarget->hasNEON())) {
4051 return fastEmitInst_r(AArch64::FSQRTv2f64, &AArch64::FPR128RegClass, Op0, Op0IsKill);
4052 }
4053 return 0;
4054}
4055
4056unsigned fastEmit_ISD_FSQRT_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
4057 switch (VT.SimpleTy) {
4058 case MVT::f16: return fastEmit_ISD_FSQRT_MVT_f16_r(RetVT, Op0, Op0IsKill);
4059 case MVT::f32: return fastEmit_ISD_FSQRT_MVT_f32_r(RetVT, Op0, Op0IsKill);
4060 case MVT::f64: return fastEmit_ISD_FSQRT_MVT_f64_r(RetVT, Op0, Op0IsKill);
4061 case MVT::v4f16: return fastEmit_ISD_FSQRT_MVT_v4f16_r(RetVT, Op0, Op0IsKill);
4062 case MVT::v8f16: return fastEmit_ISD_FSQRT_MVT_v8f16_r(RetVT, Op0, Op0IsKill);
4063 case MVT::v2f32: return fastEmit_ISD_FSQRT_MVT_v2f32_r(RetVT, Op0, Op0IsKill);
4064 case MVT::v4f32: return fastEmit_ISD_FSQRT_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
4065 case MVT::v2f64: return fastEmit_ISD_FSQRT_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
4066 default: return 0;
4067 }
4068}
4069
4070// FastEmit functions for ISD::FTRUNC.
4071
4072unsigned fastEmit_ISD_FTRUNC_MVT_f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4073 if (RetVT.SimpleTy != MVT::f16)
4074 return 0;
4075 if ((Subtarget->hasFullFP16())) {
4076 return fastEmitInst_r(AArch64::FRINTZHr, &AArch64::FPR16RegClass, Op0, Op0IsKill);
4077 }
4078 return 0;
4079}
4080
4081unsigned fastEmit_ISD_FTRUNC_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4082 if (RetVT.SimpleTy != MVT::f32)
4083 return 0;
4084 if ((Subtarget->hasFPARMv8())) {
4085 return fastEmitInst_r(AArch64::FRINTZSr, &AArch64::FPR32RegClass, Op0, Op0IsKill);
4086 }
4087 return 0;
4088}
4089
4090unsigned fastEmit_ISD_FTRUNC_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4091 if (RetVT.SimpleTy != MVT::f64)
4092 return 0;
4093 if ((Subtarget->hasFPARMv8())) {
4094 return fastEmitInst_r(AArch64::FRINTZDr, &AArch64::FPR64RegClass, Op0, Op0IsKill);
4095 }
4096 return 0;
4097}
4098
4099unsigned fastEmit_ISD_FTRUNC_MVT_v4f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4100 if (RetVT.SimpleTy != MVT::v4f16)
4101 return 0;
4102 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
4103 return fastEmitInst_r(AArch64::FRINTZv4f16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
4104 }
4105 return 0;
4106}
4107
4108unsigned fastEmit_ISD_FTRUNC_MVT_v8f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4109 if (RetVT.SimpleTy != MVT::v8f16)
4110 return 0;
4111 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
4112 return fastEmitInst_r(AArch64::FRINTZv8f16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
4113 }
4114 return 0;
4115}
4116
4117unsigned fastEmit_ISD_FTRUNC_MVT_v2f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4118 if (RetVT.SimpleTy != MVT::v2f32)
4119 return 0;
4120 if ((Subtarget->hasNEON())) {
4121 return fastEmitInst_r(AArch64::FRINTZv2f32, &AArch64::FPR64RegClass, Op0, Op0IsKill);
4122 }
4123 return 0;
4124}
4125
4126unsigned fastEmit_ISD_FTRUNC_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4127 if (RetVT.SimpleTy != MVT::v4f32)
4128 return 0;
4129 if ((Subtarget->hasNEON())) {
4130 return fastEmitInst_r(AArch64::FRINTZv4f32, &AArch64::FPR128RegClass, Op0, Op0IsKill);
4131 }
4132 return 0;
4133}
4134
4135unsigned fastEmit_ISD_FTRUNC_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4136 if (RetVT.SimpleTy != MVT::v2f64)
4137 return 0;
4138 if ((Subtarget->hasNEON())) {
4139 return fastEmitInst_r(AArch64::FRINTZv2f64, &AArch64::FPR128RegClass, Op0, Op0IsKill);
4140 }
4141 return 0;
4142}
4143
4144unsigned fastEmit_ISD_FTRUNC_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
4145 switch (VT.SimpleTy) {
4146 case MVT::f16: return fastEmit_ISD_FTRUNC_MVT_f16_r(RetVT, Op0, Op0IsKill);
4147 case MVT::f32: return fastEmit_ISD_FTRUNC_MVT_f32_r(RetVT, Op0, Op0IsKill);
4148 case MVT::f64: return fastEmit_ISD_FTRUNC_MVT_f64_r(RetVT, Op0, Op0IsKill);
4149 case MVT::v4f16: return fastEmit_ISD_FTRUNC_MVT_v4f16_r(RetVT, Op0, Op0IsKill);
4150 case MVT::v8f16: return fastEmit_ISD_FTRUNC_MVT_v8f16_r(RetVT, Op0, Op0IsKill);
4151 case MVT::v2f32: return fastEmit_ISD_FTRUNC_MVT_v2f32_r(RetVT, Op0, Op0IsKill);
4152 case MVT::v4f32: return fastEmit_ISD_FTRUNC_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
4153 case MVT::v2f64: return fastEmit_ISD_FTRUNC_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
4154 default: return 0;
4155 }
4156}
4157
4158// FastEmit functions for ISD::LLROUND.
4159
4160unsigned fastEmit_ISD_LLROUND_MVT_f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4161 if (RetVT.SimpleTy != MVT::i64)
4162 return 0;
4163 if ((Subtarget->hasFullFP16())) {
4164 return fastEmitInst_r(AArch64::FCVTASUXHr, &AArch64::GPR64RegClass, Op0, Op0IsKill);
4165 }
4166 return 0;
4167}
4168
4169unsigned fastEmit_ISD_LLROUND_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4170 if (RetVT.SimpleTy != MVT::i64)
4171 return 0;
4172 return fastEmitInst_r(AArch64::FCVTASUXSr, &AArch64::GPR64RegClass, Op0, Op0IsKill);
4173}
4174
4175unsigned fastEmit_ISD_LLROUND_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4176 if (RetVT.SimpleTy != MVT::i64)
4177 return 0;
4178 return fastEmitInst_r(AArch64::FCVTASUXDr, &AArch64::GPR64RegClass, Op0, Op0IsKill);
4179}
4180
4181unsigned fastEmit_ISD_LLROUND_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
4182 switch (VT.SimpleTy) {
4183 case MVT::f16: return fastEmit_ISD_LLROUND_MVT_f16_r(RetVT, Op0, Op0IsKill);
4184 case MVT::f32: return fastEmit_ISD_LLROUND_MVT_f32_r(RetVT, Op0, Op0IsKill);
4185 case MVT::f64: return fastEmit_ISD_LLROUND_MVT_f64_r(RetVT, Op0, Op0IsKill);
4186 default: return 0;
4187 }
4188}
4189
4190// FastEmit functions for ISD::LROUND.
4191
4192unsigned fastEmit_ISD_LROUND_MVT_f16_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
4193 if ((Subtarget->hasFullFP16())) {
4194 return fastEmitInst_r(AArch64::FCVTASUWHr, &AArch64::GPR32RegClass, Op0, Op0IsKill);
4195 }
4196 return 0;
4197}
4198
4199unsigned fastEmit_ISD_LROUND_MVT_f16_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
4200 if ((Subtarget->hasFullFP16())) {
4201 return fastEmitInst_r(AArch64::FCVTASUXHr, &AArch64::GPR64RegClass, Op0, Op0IsKill);
4202 }
4203 return 0;
4204}
4205
4206unsigned fastEmit_ISD_LROUND_MVT_f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4207switch (RetVT.SimpleTy) {
4208 case MVT::i32: return fastEmit_ISD_LROUND_MVT_f16_MVT_i32_r(Op0, Op0IsKill);
4209 case MVT::i64: return fastEmit_ISD_LROUND_MVT_f16_MVT_i64_r(Op0, Op0IsKill);
4210 default: return 0;
4211}
4212}
4213
4214unsigned fastEmit_ISD_LROUND_MVT_f32_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
4215 return fastEmitInst_r(AArch64::FCVTASUWSr, &AArch64::GPR32RegClass, Op0, Op0IsKill);
4216}
4217
4218unsigned fastEmit_ISD_LROUND_MVT_f32_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
4219 return fastEmitInst_r(AArch64::FCVTASUXSr, &AArch64::GPR64RegClass, Op0, Op0IsKill);
4220}
4221
4222unsigned fastEmit_ISD_LROUND_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4223switch (RetVT.SimpleTy) {
4224 case MVT::i32: return fastEmit_ISD_LROUND_MVT_f32_MVT_i32_r(Op0, Op0IsKill);
4225 case MVT::i64: return fastEmit_ISD_LROUND_MVT_f32_MVT_i64_r(Op0, Op0IsKill);
4226 default: return 0;
4227}
4228}
4229
4230unsigned fastEmit_ISD_LROUND_MVT_f64_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
4231 return fastEmitInst_r(AArch64::FCVTASUWDr, &AArch64::GPR32RegClass, Op0, Op0IsKill);
4232}
4233
4234unsigned fastEmit_ISD_LROUND_MVT_f64_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
4235 return fastEmitInst_r(AArch64::FCVTASUXDr, &AArch64::GPR64RegClass, Op0, Op0IsKill);
4236}
4237
4238unsigned fastEmit_ISD_LROUND_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4239switch (RetVT.SimpleTy) {
4240 case MVT::i32: return fastEmit_ISD_LROUND_MVT_f64_MVT_i32_r(Op0, Op0IsKill);
4241 case MVT::i64: return fastEmit_ISD_LROUND_MVT_f64_MVT_i64_r(Op0, Op0IsKill);
4242 default: return 0;
4243}
4244}
4245
4246unsigned fastEmit_ISD_LROUND_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
4247 switch (VT.SimpleTy) {
4248 case MVT::f16: return fastEmit_ISD_LROUND_MVT_f16_r(RetVT, Op0, Op0IsKill);
4249 case MVT::f32: return fastEmit_ISD_LROUND_MVT_f32_r(RetVT, Op0, Op0IsKill);
4250 case MVT::f64: return fastEmit_ISD_LROUND_MVT_f64_r(RetVT, Op0, Op0IsKill);
4251 default: return 0;
4252 }
4253}
4254
4255// FastEmit functions for ISD::SINT_TO_FP.
4256
4257unsigned fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f16_r(unsigned Op0, bool Op0IsKill) {
4258 if ((Subtarget->hasFullFP16())) {
4259 return fastEmitInst_r(AArch64::SCVTFUWHri, &AArch64::FPR16RegClass, Op0, Op0IsKill);
4260 }
4261 return 0;
4262}
4263
4264unsigned fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f32_r(unsigned Op0, bool Op0IsKill) {
4265 if ((Subtarget->hasFPARMv8())) {
4266 return fastEmitInst_r(AArch64::SCVTFUWSri, &AArch64::FPR32RegClass, Op0, Op0IsKill);
4267 }
4268 return 0;
4269}
4270
4271unsigned fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f64_r(unsigned Op0, bool Op0IsKill) {
4272 if ((Subtarget->hasFPARMv8())) {
4273 return fastEmitInst_r(AArch64::SCVTFUWDri, &AArch64::FPR64RegClass, Op0, Op0IsKill);
4274 }
4275 return 0;
4276}
4277
4278unsigned fastEmit_ISD_SINT_TO_FP_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4279switch (RetVT.SimpleTy) {
4280 case MVT::f16: return fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f16_r(Op0, Op0IsKill);
4281 case MVT::f32: return fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f32_r(Op0, Op0IsKill);
4282 case MVT::f64: return fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f64_r(Op0, Op0IsKill);
4283 default: return 0;
4284}
4285}
4286
4287unsigned fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f16_r(unsigned Op0, bool Op0IsKill) {
4288 if ((Subtarget->hasFullFP16())) {
4289 return fastEmitInst_r(AArch64::SCVTFUXHri, &AArch64::FPR16RegClass, Op0, Op0IsKill);
4290 }
4291 return 0;
4292}
4293
4294unsigned fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f32_r(unsigned Op0, bool Op0IsKill) {
4295 if ((Subtarget->hasFPARMv8())) {
4296 return fastEmitInst_r(AArch64::SCVTFUXSri, &AArch64::FPR32RegClass, Op0, Op0IsKill);
4297 }
4298 return 0;
4299}
4300
4301unsigned fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f64_r(unsigned Op0, bool Op0IsKill) {
4302 if ((Subtarget->hasFPARMv8())) {
4303 return fastEmitInst_r(AArch64::SCVTFUXDri, &AArch64::FPR64RegClass, Op0, Op0IsKill);
4304 }
4305 return 0;
4306}
4307
4308unsigned fastEmit_ISD_SINT_TO_FP_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4309switch (RetVT.SimpleTy) {
4310 case MVT::f16: return fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f16_r(Op0, Op0IsKill);
4311 case MVT::f32: return fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f32_r(Op0, Op0IsKill);
4312 case MVT::f64: return fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f64_r(Op0, Op0IsKill);
4313 default: return 0;
4314}
4315}
4316
4317unsigned fastEmit_ISD_SINT_TO_FP_MVT_v4i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4318 if (RetVT.SimpleTy != MVT::v4f16)
4319 return 0;
4320 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
4321 return fastEmitInst_r(AArch64::SCVTFv4f16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
4322 }
4323 return 0;
4324}
4325
4326unsigned fastEmit_ISD_SINT_TO_FP_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4327 if (RetVT.SimpleTy != MVT::v8f16)
4328 return 0;
4329 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
4330 return fastEmitInst_r(AArch64::SCVTFv8f16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
4331 }
4332 return 0;
4333}
4334
4335unsigned fastEmit_ISD_SINT_TO_FP_MVT_v2i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4336 if (RetVT.SimpleTy != MVT::v2f32)
4337 return 0;
4338 if ((Subtarget->hasNEON())) {
4339 return fastEmitInst_r(AArch64::SCVTFv2f32, &AArch64::FPR64RegClass, Op0, Op0IsKill);
4340 }
4341 return 0;
4342}
4343
4344unsigned fastEmit_ISD_SINT_TO_FP_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4345 if (RetVT.SimpleTy != MVT::v4f32)
4346 return 0;
4347 if ((Subtarget->hasNEON())) {
4348 return fastEmitInst_r(AArch64::SCVTFv4f32, &AArch64::FPR128RegClass, Op0, Op0IsKill);
4349 }
4350 return 0;
4351}
4352
4353unsigned fastEmit_ISD_SINT_TO_FP_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4354 if (RetVT.SimpleTy != MVT::v2f64)
4355 return 0;
4356 if ((Subtarget->hasNEON())) {
4357 return fastEmitInst_r(AArch64::SCVTFv2f64, &AArch64::FPR128RegClass, Op0, Op0IsKill);
4358 }
4359 return 0;
4360}
4361
4362unsigned fastEmit_ISD_SINT_TO_FP_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
4363 switch (VT.SimpleTy) {
4364 case MVT::i32: return fastEmit_ISD_SINT_TO_FP_MVT_i32_r(RetVT, Op0, Op0IsKill);
4365 case MVT::i64: return fastEmit_ISD_SINT_TO_FP_MVT_i64_r(RetVT, Op0, Op0IsKill);
4366 case MVT::v4i16: return fastEmit_ISD_SINT_TO_FP_MVT_v4i16_r(RetVT, Op0, Op0IsKill);
4367 case MVT::v8i16: return fastEmit_ISD_SINT_TO_FP_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
4368 case MVT::v2i32: return fastEmit_ISD_SINT_TO_FP_MVT_v2i32_r(RetVT, Op0, Op0IsKill);
4369 case MVT::v4i32: return fastEmit_ISD_SINT_TO_FP_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
4370 case MVT::v2i64: return fastEmit_ISD_SINT_TO_FP_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
4371 default: return 0;
4372 }
4373}
4374
4375// FastEmit functions for ISD::STRICT_FP_ROUND.
4376
4377unsigned fastEmit_ISD_STRICT_FP_ROUND_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4378 if (RetVT.SimpleTy != MVT::f16)
4379 return 0;
4380 if ((Subtarget->hasFPARMv8())) {
4381 return fastEmitInst_r(AArch64::FCVTHSr, &AArch64::FPR16RegClass, Op0, Op0IsKill);
4382 }
4383 return 0;
4384}
4385
4386unsigned fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_MVT_f16_r(unsigned Op0, bool Op0IsKill) {
4387 if ((Subtarget->hasFPARMv8())) {
4388 return fastEmitInst_r(AArch64::FCVTHDr, &AArch64::FPR16RegClass, Op0, Op0IsKill);
4389 }
4390 return 0;
4391}
4392
4393unsigned fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_MVT_f32_r(unsigned Op0, bool Op0IsKill) {
4394 if ((Subtarget->hasFPARMv8())) {
4395 return fastEmitInst_r(AArch64::FCVTSDr, &AArch64::FPR32RegClass, Op0, Op0IsKill);
4396 }
4397 return 0;
4398}
4399
4400unsigned fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4401switch (RetVT.SimpleTy) {
4402 case MVT::f16: return fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_MVT_f16_r(Op0, Op0IsKill);
4403 case MVT::f32: return fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_MVT_f32_r(Op0, Op0IsKill);
4404 default: return 0;
4405}
4406}
4407
4408unsigned fastEmit_ISD_STRICT_FP_ROUND_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
4409 switch (VT.SimpleTy) {
4410 case MVT::f32: return fastEmit_ISD_STRICT_FP_ROUND_MVT_f32_r(RetVT, Op0, Op0IsKill);
4411 case MVT::f64: return fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_r(RetVT, Op0, Op0IsKill);
4412 default: return 0;
4413 }
4414}
4415
4416// FastEmit functions for ISD::STRICT_FP_TO_SINT.
4417
4418unsigned fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
4419 if ((Subtarget->hasFullFP16())) {
4420 return fastEmitInst_r(AArch64::FCVTZSUWHr, &AArch64::GPR32RegClass, Op0, Op0IsKill);
4421 }
4422 return 0;
4423}
4424
4425unsigned fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
4426 if ((Subtarget->hasFullFP16())) {
4427 return fastEmitInst_r(AArch64::FCVTZSUXHr, &AArch64::GPR64RegClass, Op0, Op0IsKill);
4428 }
4429 return 0;
4430}
4431
4432unsigned fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4433switch (RetVT.SimpleTy) {
4434 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_MVT_i32_r(Op0, Op0IsKill);
4435 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_MVT_i64_r(Op0, Op0IsKill);
4436 default: return 0;
4437}
4438}
4439
4440unsigned fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
4441 if ((Subtarget->hasFPARMv8())) {
4442 return fastEmitInst_r(AArch64::FCVTZSUWSr, &AArch64::GPR32RegClass, Op0, Op0IsKill);
4443 }
4444 return 0;
4445}
4446
4447unsigned fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
4448 if ((Subtarget->hasFPARMv8())) {
4449 return fastEmitInst_r(AArch64::FCVTZSUXSr, &AArch64::GPR64RegClass, Op0, Op0IsKill);
4450 }
4451 return 0;
4452}
4453
4454unsigned fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4455switch (RetVT.SimpleTy) {
4456 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_MVT_i32_r(Op0, Op0IsKill);
4457 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_MVT_i64_r(Op0, Op0IsKill);
4458 default: return 0;
4459}
4460}
4461
4462unsigned fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
4463 if ((Subtarget->hasFPARMv8())) {
4464 return fastEmitInst_r(AArch64::FCVTZSUWDr, &AArch64::GPR32RegClass, Op0, Op0IsKill);
4465 }
4466 return 0;
4467}
4468
4469unsigned fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
4470 if ((Subtarget->hasFPARMv8())) {
4471 return fastEmitInst_r(AArch64::FCVTZSUXDr, &AArch64::GPR64RegClass, Op0, Op0IsKill);
4472 }
4473 return 0;
4474}
4475
4476unsigned fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4477switch (RetVT.SimpleTy) {
4478 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_MVT_i32_r(Op0, Op0IsKill);
4479 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_MVT_i64_r(Op0, Op0IsKill);
4480 default: return 0;
4481}
4482}
4483
4484unsigned fastEmit_ISD_STRICT_FP_TO_SINT_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
4485 switch (VT.SimpleTy) {
4486 case MVT::f16: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_r(RetVT, Op0, Op0IsKill);
4487 case MVT::f32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_r(RetVT, Op0, Op0IsKill);
4488 case MVT::f64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_r(RetVT, Op0, Op0IsKill);
4489 default: return 0;
4490 }
4491}
4492
4493// FastEmit functions for ISD::STRICT_FP_TO_UINT.
4494
4495unsigned fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
4496 if ((Subtarget->hasFullFP16())) {
4497 return fastEmitInst_r(AArch64::FCVTZUUWHr, &AArch64::GPR32RegClass, Op0, Op0IsKill);
4498 }
4499 return 0;
4500}
4501
4502unsigned fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
4503 if ((Subtarget->hasFullFP16())) {
4504 return fastEmitInst_r(AArch64::FCVTZUUXHr, &AArch64::GPR64RegClass, Op0, Op0IsKill);
4505 }
4506 return 0;
4507}
4508
4509unsigned fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4510switch (RetVT.SimpleTy) {
4511 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_MVT_i32_r(Op0, Op0IsKill);
4512 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_MVT_i64_r(Op0, Op0IsKill);
4513 default: return 0;
4514}
4515}
4516
4517unsigned fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
4518 if ((Subtarget->hasFPARMv8())) {
4519 return fastEmitInst_r(AArch64::FCVTZUUWSr, &AArch64::GPR32RegClass, Op0, Op0IsKill);
4520 }
4521 return 0;
4522}
4523
4524unsigned fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
4525 if ((Subtarget->hasFPARMv8())) {
4526 return fastEmitInst_r(AArch64::FCVTZUUXSr, &AArch64::GPR64RegClass, Op0, Op0IsKill);
4527 }
4528 return 0;
4529}
4530
4531unsigned fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4532switch (RetVT.SimpleTy) {
4533 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_MVT_i32_r(Op0, Op0IsKill);
4534 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_MVT_i64_r(Op0, Op0IsKill);
4535 default: return 0;
4536}
4537}
4538
4539unsigned fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
4540 if ((Subtarget->hasFPARMv8())) {
4541 return fastEmitInst_r(AArch64::FCVTZUUWDr, &AArch64::GPR32RegClass, Op0, Op0IsKill);
4542 }
4543 return 0;
4544}
4545
4546unsigned fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
4547 if ((Subtarget->hasFPARMv8())) {
4548 return fastEmitInst_r(AArch64::FCVTZUUXDr, &AArch64::GPR64RegClass, Op0, Op0IsKill);
4549 }
4550 return 0;
4551}
4552
4553unsigned fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4554switch (RetVT.SimpleTy) {
4555 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_MVT_i32_r(Op0, Op0IsKill);
4556 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_MVT_i64_r(Op0, Op0IsKill);
4557 default: return 0;
4558}
4559}
4560
4561unsigned fastEmit_ISD_STRICT_FP_TO_UINT_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
4562 switch (VT.SimpleTy) {
4563 case MVT::f16: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_r(RetVT, Op0, Op0IsKill);
4564 case MVT::f32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_r(RetVT, Op0, Op0IsKill);
4565 case MVT::f64: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_r(RetVT, Op0, Op0IsKill);
4566 default: return 0;
4567 }
4568}
4569
4570// FastEmit functions for ISD::STRICT_SINT_TO_FP.
4571
4572unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f16_r(unsigned Op0, bool Op0IsKill) {
4573 if ((Subtarget->hasFullFP16())) {
4574 return fastEmitInst_r(AArch64::SCVTFUWHri, &AArch64::FPR16RegClass, Op0, Op0IsKill);
4575 }
4576 return 0;
4577}
4578
4579unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f32_r(unsigned Op0, bool Op0IsKill) {
4580 if ((Subtarget->hasFPARMv8())) {
4581 return fastEmitInst_r(AArch64::SCVTFUWSri, &AArch64::FPR32RegClass, Op0, Op0IsKill);
4582 }
4583 return 0;
4584}
4585
4586unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f64_r(unsigned Op0, bool Op0IsKill) {
4587 if ((Subtarget->hasFPARMv8())) {
4588 return fastEmitInst_r(AArch64::SCVTFUWDri, &AArch64::FPR64RegClass, Op0, Op0IsKill);
4589 }
4590 return 0;
4591}
4592
4593unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4594switch (RetVT.SimpleTy) {
4595 case MVT::f16: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f16_r(Op0, Op0IsKill);
4596 case MVT::f32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f32_r(Op0, Op0IsKill);
4597 case MVT::f64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f64_r(Op0, Op0IsKill);
4598 default: return 0;
4599}
4600}
4601
4602unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f16_r(unsigned Op0, bool Op0IsKill) {
4603 if ((Subtarget->hasFullFP16())) {
4604 return fastEmitInst_r(AArch64::SCVTFUXHri, &AArch64::FPR16RegClass, Op0, Op0IsKill);
4605 }
4606 return 0;
4607}
4608
4609unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f32_r(unsigned Op0, bool Op0IsKill) {
4610 if ((Subtarget->hasFPARMv8())) {
4611 return fastEmitInst_r(AArch64::SCVTFUXSri, &AArch64::FPR32RegClass, Op0, Op0IsKill);
4612 }
4613 return 0;
4614}
4615
4616unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f64_r(unsigned Op0, bool Op0IsKill) {
4617 if ((Subtarget->hasFPARMv8())) {
4618 return fastEmitInst_r(AArch64::SCVTFUXDri, &AArch64::FPR64RegClass, Op0, Op0IsKill);
4619 }
4620 return 0;
4621}
4622
4623unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4624switch (RetVT.SimpleTy) {
4625 case MVT::f16: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f16_r(Op0, Op0IsKill);
4626 case MVT::f32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f32_r(Op0, Op0IsKill);
4627 case MVT::f64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f64_r(Op0, Op0IsKill);
4628 default: return 0;
4629}
4630}
4631
4632unsigned fastEmit_ISD_STRICT_SINT_TO_FP_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
4633 switch (VT.SimpleTy) {
4634 case MVT::i32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_r(RetVT, Op0, Op0IsKill);
4635 case MVT::i64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_r(RetVT, Op0, Op0IsKill);
4636 default: return 0;
4637 }
4638}
4639
4640// FastEmit functions for ISD::STRICT_UINT_TO_FP.
4641
4642unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i32_MVT_f16_r(unsigned Op0, bool Op0IsKill) {
4643 if ((Subtarget->hasFullFP16())) {
4644 return fastEmitInst_r(AArch64::UCVTFUWHri, &AArch64::FPR16RegClass, Op0, Op0IsKill);
4645 }
4646 return 0;
4647}
4648
4649unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i32_MVT_f32_r(unsigned Op0, bool Op0IsKill) {
4650 if ((Subtarget->hasFPARMv8())) {
4651 return fastEmitInst_r(AArch64::UCVTFUWSri, &AArch64::FPR32RegClass, Op0, Op0IsKill);
4652 }
4653 return 0;
4654}
4655
4656unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i32_MVT_f64_r(unsigned Op0, bool Op0IsKill) {
4657 if ((Subtarget->hasFPARMv8())) {
4658 return fastEmitInst_r(AArch64::UCVTFUWDri, &AArch64::FPR64RegClass, Op0, Op0IsKill);
4659 }
4660 return 0;
4661}
4662
4663unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4664switch (RetVT.SimpleTy) {
4665 case MVT::f16: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i32_MVT_f16_r(Op0, Op0IsKill);
4666 case MVT::f32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i32_MVT_f32_r(Op0, Op0IsKill);
4667 case MVT::f64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i32_MVT_f64_r(Op0, Op0IsKill);
4668 default: return 0;
4669}
4670}
4671
4672unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i64_MVT_f16_r(unsigned Op0, bool Op0IsKill) {
4673 if ((Subtarget->hasFullFP16())) {
4674 return fastEmitInst_r(AArch64::UCVTFUXHri, &AArch64::FPR16RegClass, Op0, Op0IsKill);
4675 }
4676 return 0;
4677}
4678
4679unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i64_MVT_f32_r(unsigned Op0, bool Op0IsKill) {
4680 if ((Subtarget->hasFPARMv8())) {
4681 return fastEmitInst_r(AArch64::UCVTFUXSri, &AArch64::FPR32RegClass, Op0, Op0IsKill);
4682 }
4683 return 0;
4684}
4685
4686unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i64_MVT_f64_r(unsigned Op0, bool Op0IsKill) {
4687 if ((Subtarget->hasFPARMv8())) {
4688 return fastEmitInst_r(AArch64::UCVTFUXDri, &AArch64::FPR64RegClass, Op0, Op0IsKill);
4689 }
4690 return 0;
4691}
4692
4693unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4694switch (RetVT.SimpleTy) {
4695 case MVT::f16: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i64_MVT_f16_r(Op0, Op0IsKill);
4696 case MVT::f32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i64_MVT_f32_r(Op0, Op0IsKill);
4697 case MVT::f64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i64_MVT_f64_r(Op0, Op0IsKill);
4698 default: return 0;
4699}
4700}
4701
4702unsigned fastEmit_ISD_STRICT_UINT_TO_FP_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
4703 switch (VT.SimpleTy) {
4704 case MVT::i32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i32_r(RetVT, Op0, Op0IsKill);
4705 case MVT::i64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i64_r(RetVT, Op0, Op0IsKill);
4706 default: return 0;
4707 }
4708}
4709
4710// FastEmit functions for ISD::TRUNCATE.
4711
4712unsigned fastEmit_ISD_TRUNCATE_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4713 if (RetVT.SimpleTy != MVT::i32)
4714 return 0;
4715 return fastEmitInst_extractsubreg(RetVT, Op0, Op0IsKill, AArch64::sub_32);
4716}
4717
4718unsigned fastEmit_ISD_TRUNCATE_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4719 if (RetVT.SimpleTy != MVT::v8i8)
4720 return 0;
4721 if ((Subtarget->hasNEON())) {
4722 return fastEmitInst_r(AArch64::XTNv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill);
4723 }
4724 return 0;
4725}
4726
4727unsigned fastEmit_ISD_TRUNCATE_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4728 if (RetVT.SimpleTy != MVT::v4i16)
4729 return 0;
4730 if ((Subtarget->hasNEON())) {
4731 return fastEmitInst_r(AArch64::XTNv4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
4732 }
4733 return 0;
4734}
4735
4736unsigned fastEmit_ISD_TRUNCATE_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4737 if (RetVT.SimpleTy != MVT::v2i32)
4738 return 0;
4739 if ((Subtarget->hasNEON())) {
4740 return fastEmitInst_r(AArch64::XTNv2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill);
4741 }
4742 return 0;
4743}
4744
4745unsigned fastEmit_ISD_TRUNCATE_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
4746 switch (VT.SimpleTy) {
4747 case MVT::i64: return fastEmit_ISD_TRUNCATE_MVT_i64_r(RetVT, Op0, Op0IsKill);
4748 case MVT::v8i16: return fastEmit_ISD_TRUNCATE_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
4749 case MVT::v4i32: return fastEmit_ISD_TRUNCATE_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
4750 case MVT::v2i64: return fastEmit_ISD_TRUNCATE_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
4751 default: return 0;
4752 }
4753}
4754
4755// FastEmit functions for ISD::UINT_TO_FP.
4756
4757unsigned fastEmit_ISD_UINT_TO_FP_MVT_i32_MVT_f16_r(unsigned Op0, bool Op0IsKill) {
4758 if ((Subtarget->hasFullFP16())) {
4759 return fastEmitInst_r(AArch64::UCVTFUWHri, &AArch64::FPR16RegClass, Op0, Op0IsKill);
4760 }
4761 return 0;
4762}
4763
4764unsigned fastEmit_ISD_UINT_TO_FP_MVT_i32_MVT_f32_r(unsigned Op0, bool Op0IsKill) {
4765 if ((Subtarget->hasFPARMv8())) {
4766 return fastEmitInst_r(AArch64::UCVTFUWSri, &AArch64::FPR32RegClass, Op0, Op0IsKill);
4767 }
4768 return 0;
4769}
4770
4771unsigned fastEmit_ISD_UINT_TO_FP_MVT_i32_MVT_f64_r(unsigned Op0, bool Op0IsKill) {
4772 if ((Subtarget->hasFPARMv8())) {
4773 return fastEmitInst_r(AArch64::UCVTFUWDri, &AArch64::FPR64RegClass, Op0, Op0IsKill);
4774 }
4775 return 0;
4776}
4777
4778unsigned fastEmit_ISD_UINT_TO_FP_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4779switch (RetVT.SimpleTy) {
4780 case MVT::f16: return fastEmit_ISD_UINT_TO_FP_MVT_i32_MVT_f16_r(Op0, Op0IsKill);
4781 case MVT::f32: return fastEmit_ISD_UINT_TO_FP_MVT_i32_MVT_f32_r(Op0, Op0IsKill);
4782 case MVT::f64: return fastEmit_ISD_UINT_TO_FP_MVT_i32_MVT_f64_r(Op0, Op0IsKill);
4783 default: return 0;
4784}
4785}
4786
4787unsigned fastEmit_ISD_UINT_TO_FP_MVT_i64_MVT_f16_r(unsigned Op0, bool Op0IsKill) {
4788 if ((Subtarget->hasFullFP16())) {
4789 return fastEmitInst_r(AArch64::UCVTFUXHri, &AArch64::FPR16RegClass, Op0, Op0IsKill);
4790 }
4791 return 0;
4792}
4793
4794unsigned fastEmit_ISD_UINT_TO_FP_MVT_i64_MVT_f32_r(unsigned Op0, bool Op0IsKill) {
4795 if ((Subtarget->hasFPARMv8())) {
4796 return fastEmitInst_r(AArch64::UCVTFUXSri, &AArch64::FPR32RegClass, Op0, Op0IsKill);
4797 }
4798 return 0;
4799}
4800
4801unsigned fastEmit_ISD_UINT_TO_FP_MVT_i64_MVT_f64_r(unsigned Op0, bool Op0IsKill) {
4802 if ((Subtarget->hasFPARMv8())) {
4803 return fastEmitInst_r(AArch64::UCVTFUXDri, &AArch64::FPR64RegClass, Op0, Op0IsKill);
4804 }
4805 return 0;
4806}
4807
4808unsigned fastEmit_ISD_UINT_TO_FP_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4809switch (RetVT.SimpleTy) {
4810 case MVT::f16: return fastEmit_ISD_UINT_TO_FP_MVT_i64_MVT_f16_r(Op0, Op0IsKill);
4811 case MVT::f32: return fastEmit_ISD_UINT_TO_FP_MVT_i64_MVT_f32_r(Op0, Op0IsKill);
4812 case MVT::f64: return fastEmit_ISD_UINT_TO_FP_MVT_i64_MVT_f64_r(Op0, Op0IsKill);
4813 default: return 0;
4814}
4815}
4816
4817unsigned fastEmit_ISD_UINT_TO_FP_MVT_v4i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4818 if (RetVT.SimpleTy != MVT::v4f16)
4819 return 0;
4820 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
4821 return fastEmitInst_r(AArch64::UCVTFv4f16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
4822 }
4823 return 0;
4824}
4825
4826unsigned fastEmit_ISD_UINT_TO_FP_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4827 if (RetVT.SimpleTy != MVT::v8f16)
4828 return 0;
4829 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
4830 return fastEmitInst_r(AArch64::UCVTFv8f16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
4831 }
4832 return 0;
4833}
4834
4835unsigned fastEmit_ISD_UINT_TO_FP_MVT_v2i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4836 if (RetVT.SimpleTy != MVT::v2f32)
4837 return 0;
4838 if ((Subtarget->hasNEON())) {
4839 return fastEmitInst_r(AArch64::UCVTFv2f32, &AArch64::FPR64RegClass, Op0, Op0IsKill);
4840 }
4841 return 0;
4842}
4843
4844unsigned fastEmit_ISD_UINT_TO_FP_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4845 if (RetVT.SimpleTy != MVT::v4f32)
4846 return 0;
4847 if ((Subtarget->hasNEON())) {
4848 return fastEmitInst_r(AArch64::UCVTFv4f32, &AArch64::FPR128RegClass, Op0, Op0IsKill);
4849 }
4850 return 0;
4851}
4852
4853unsigned fastEmit_ISD_UINT_TO_FP_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4854 if (RetVT.SimpleTy != MVT::v2f64)
4855 return 0;
4856 if ((Subtarget->hasNEON())) {
4857 return fastEmitInst_r(AArch64::UCVTFv2f64, &AArch64::FPR128RegClass, Op0, Op0IsKill);
4858 }
4859 return 0;
4860}
4861
4862unsigned fastEmit_ISD_UINT_TO_FP_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
4863 switch (VT.SimpleTy) {
4864 case MVT::i32: return fastEmit_ISD_UINT_TO_FP_MVT_i32_r(RetVT, Op0, Op0IsKill);
4865 case MVT::i64: return fastEmit_ISD_UINT_TO_FP_MVT_i64_r(RetVT, Op0, Op0IsKill);
4866 case MVT::v4i16: return fastEmit_ISD_UINT_TO_FP_MVT_v4i16_r(RetVT, Op0, Op0IsKill);
4867 case MVT::v8i16: return fastEmit_ISD_UINT_TO_FP_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
4868 case MVT::v2i32: return fastEmit_ISD_UINT_TO_FP_MVT_v2i32_r(RetVT, Op0, Op0IsKill);
4869 case MVT::v4i32: return fastEmit_ISD_UINT_TO_FP_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
4870 case MVT::v2i64: return fastEmit_ISD_UINT_TO_FP_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
4871 default: return 0;
4872 }
4873}
4874
4875// FastEmit functions for ISD::VECREDUCE_FADD.
4876
4877unsigned fastEmit_ISD_VECREDUCE_FADD_MVT_v2f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4878 if (RetVT.SimpleTy != MVT::f32)
4879 return 0;
4880 return fastEmitInst_r(AArch64::FADDPv2i32p, &AArch64::FPR32RegClass, Op0, Op0IsKill);
4881}
4882
4883unsigned fastEmit_ISD_VECREDUCE_FADD_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4884 if (RetVT.SimpleTy != MVT::f64)
4885 return 0;
4886 return fastEmitInst_r(AArch64::FADDPv2i64p, &AArch64::FPR64RegClass, Op0, Op0IsKill);
4887}
4888
4889unsigned fastEmit_ISD_VECREDUCE_FADD_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
4890 switch (VT.SimpleTy) {
4891 case MVT::v2f32: return fastEmit_ISD_VECREDUCE_FADD_MVT_v2f32_r(RetVT, Op0, Op0IsKill);
4892 case MVT::v2f64: return fastEmit_ISD_VECREDUCE_FADD_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
4893 default: return 0;
4894 }
4895}
4896
4897// Top-level FastEmit function.
4898
4899unsigned fastEmit_r(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill) override {
4900 switch (Opcode) {
4901 case AArch64ISD::CALL: return fastEmit_AArch64ISD_CALL_r(VT, RetVT, Op0, Op0IsKill);
4902 case AArch64ISD::CMEQz: return fastEmit_AArch64ISD_CMEQz_r(VT, RetVT, Op0, Op0IsKill);
4903 case AArch64ISD::CMGEz: return fastEmit_AArch64ISD_CMGEz_r(VT, RetVT, Op0, Op0IsKill);
4904 case AArch64ISD::CMGTz: return fastEmit_AArch64ISD_CMGTz_r(VT, RetVT, Op0, Op0IsKill);
4905 case AArch64ISD::CMLEz: return fastEmit_AArch64ISD_CMLEz_r(VT, RetVT, Op0, Op0IsKill);
4906 case AArch64ISD::CMLTz: return fastEmit_AArch64ISD_CMLTz_r(VT, RetVT, Op0, Op0IsKill);
4907 case AArch64ISD::DUP: return fastEmit_AArch64ISD_DUP_r(VT, RetVT, Op0, Op0IsKill);
4908 case AArch64ISD::FCMEQz: return fastEmit_AArch64ISD_FCMEQz_r(VT, RetVT, Op0, Op0IsKill);
4909 case AArch64ISD::FCMGEz: return fastEmit_AArch64ISD_FCMGEz_r(VT, RetVT, Op0, Op0IsKill);
4910 case AArch64ISD::FCMGTz: return fastEmit_AArch64ISD_FCMGTz_r(VT, RetVT, Op0, Op0IsKill);
4911 case AArch64ISD::FCMLEz: return fastEmit_AArch64ISD_FCMLEz_r(VT, RetVT, Op0, Op0IsKill);
4912 case AArch64ISD::FCMLTz: return fastEmit_AArch64ISD_FCMLTz_r(VT, RetVT, Op0, Op0IsKill);
4913 case AArch64ISD::FRECPE: return fastEmit_AArch64ISD_FRECPE_r(VT, RetVT, Op0, Op0IsKill);
4914 case AArch64ISD::FRSQRTE: return fastEmit_AArch64ISD_FRSQRTE_r(VT, RetVT, Op0, Op0IsKill);
4915 case AArch64ISD::NEG: return fastEmit_AArch64ISD_NEG_r(VT, RetVT, Op0, Op0IsKill);
4916 case AArch64ISD::REV: return fastEmit_AArch64ISD_REV_r(VT, RetVT, Op0, Op0IsKill);
4917 case AArch64ISD::REV16: return fastEmit_AArch64ISD_REV16_r(VT, RetVT, Op0, Op0IsKill);
4918 case AArch64ISD::REV32: return fastEmit_AArch64ISD_REV32_r(VT, RetVT, Op0, Op0IsKill);
4919 case AArch64ISD::REV64: return fastEmit_AArch64ISD_REV64_r(VT, RetVT, Op0, Op0IsKill);
4920 case AArch64ISD::SITOF: return fastEmit_AArch64ISD_SITOF_r(VT, RetVT, Op0, Op0IsKill);
4921 case AArch64ISD::SUNPKHI: return fastEmit_AArch64ISD_SUNPKHI_r(VT, RetVT, Op0, Op0IsKill);
4922 case AArch64ISD::SUNPKLO: return fastEmit_AArch64ISD_SUNPKLO_r(VT, RetVT, Op0, Op0IsKill);
4923 case AArch64ISD::UITOF: return fastEmit_AArch64ISD_UITOF_r(VT, RetVT, Op0, Op0IsKill);
4924 case AArch64ISD::UUNPKHI: return fastEmit_AArch64ISD_UUNPKHI_r(VT, RetVT, Op0, Op0IsKill);
4925 case AArch64ISD::UUNPKLO: return fastEmit_AArch64ISD_UUNPKLO_r(VT, RetVT, Op0, Op0IsKill);
4926 case ISD::ABS: return fastEmit_ISD_ABS_r(VT, RetVT, Op0, Op0IsKill);
4927 case ISD::BITCAST: return fastEmit_ISD_BITCAST_r(VT, RetVT, Op0, Op0IsKill);
4928 case ISD::BITREVERSE: return fastEmit_ISD_BITREVERSE_r(VT, RetVT, Op0, Op0IsKill);
4929 case ISD::BRIND: return fastEmit_ISD_BRIND_r(VT, RetVT, Op0, Op0IsKill);
4930 case ISD::BSWAP: return fastEmit_ISD_BSWAP_r(VT, RetVT, Op0, Op0IsKill);
4931 case ISD::CTLZ: return fastEmit_ISD_CTLZ_r(VT, RetVT, Op0, Op0IsKill);
4932 case ISD::CTPOP: return fastEmit_ISD_CTPOP_r(VT, RetVT, Op0, Op0IsKill);
4933 case ISD::FABS: return fastEmit_ISD_FABS_r(VT, RetVT, Op0, Op0IsKill);
4934 case ISD::FCEIL: return fastEmit_ISD_FCEIL_r(VT, RetVT, Op0, Op0IsKill);
4935 case ISD::FFLOOR: return fastEmit_ISD_FFLOOR_r(VT, RetVT, Op0, Op0IsKill);
4936 case ISD::FNEARBYINT: return fastEmit_ISD_FNEARBYINT_r(VT, RetVT, Op0, Op0IsKill);
4937 case ISD::FNEG: return fastEmit_ISD_FNEG_r(VT, RetVT, Op0, Op0IsKill);
4938 case ISD::FP_EXTEND: return fastEmit_ISD_FP_EXTEND_r(VT, RetVT, Op0, Op0IsKill);
4939 case ISD::FP_ROUND: return fastEmit_ISD_FP_ROUND_r(VT, RetVT, Op0, Op0IsKill);
4940 case ISD::FP_TO_SINT: return fastEmit_ISD_FP_TO_SINT_r(VT, RetVT, Op0, Op0IsKill);
4941 case ISD::FP_TO_UINT: return fastEmit_ISD_FP_TO_UINT_r(VT, RetVT, Op0, Op0IsKill);
4942 case ISD::FRINT: return fastEmit_ISD_FRINT_r(VT, RetVT, Op0, Op0IsKill);
4943 case ISD::FROUND: return fastEmit_ISD_FROUND_r(VT, RetVT, Op0, Op0IsKill);
4944 case ISD::FSQRT: return fastEmit_ISD_FSQRT_r(VT, RetVT, Op0, Op0IsKill);
4945 case ISD::FTRUNC: return fastEmit_ISD_FTRUNC_r(VT, RetVT, Op0, Op0IsKill);
4946 case ISD::LLROUND: return fastEmit_ISD_LLROUND_r(VT, RetVT, Op0, Op0IsKill);
4947 case ISD::LROUND: return fastEmit_ISD_LROUND_r(VT, RetVT, Op0, Op0IsKill);
4948 case ISD::SINT_TO_FP: return fastEmit_ISD_SINT_TO_FP_r(VT, RetVT, Op0, Op0IsKill);
4949 case ISD::STRICT_FP_ROUND: return fastEmit_ISD_STRICT_FP_ROUND_r(VT, RetVT, Op0, Op0IsKill);
4950 case ISD::STRICT_FP_TO_SINT: return fastEmit_ISD_STRICT_FP_TO_SINT_r(VT, RetVT, Op0, Op0IsKill);
4951 case ISD::STRICT_FP_TO_UINT: return fastEmit_ISD_STRICT_FP_TO_UINT_r(VT, RetVT, Op0, Op0IsKill);
4952 case ISD::STRICT_SINT_TO_FP: return fastEmit_ISD_STRICT_SINT_TO_FP_r(VT, RetVT, Op0, Op0IsKill);
4953 case ISD::STRICT_UINT_TO_FP: return fastEmit_ISD_STRICT_UINT_TO_FP_r(VT, RetVT, Op0, Op0IsKill);
4954 case ISD::TRUNCATE: return fastEmit_ISD_TRUNCATE_r(VT, RetVT, Op0, Op0IsKill);
4955 case ISD::UINT_TO_FP: return fastEmit_ISD_UINT_TO_FP_r(VT, RetVT, Op0, Op0IsKill);
4956 case ISD::VECREDUCE_FADD: return fastEmit_ISD_VECREDUCE_FADD_r(VT, RetVT, Op0, Op0IsKill);
4957 default: return 0;
4958 }
4959}
4960
4961// FastEmit functions for AArch64ISD::CMEQ.
4962
4963unsigned fastEmit_AArch64ISD_CMEQ_MVT_v8i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
4964 if (RetVT.SimpleTy != MVT::v8i8)
4965 return 0;
4966 if ((Subtarget->hasNEON())) {
4967 return fastEmitInst_rr(AArch64::CMEQv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4968 }
4969 return 0;
4970}
4971
4972unsigned fastEmit_AArch64ISD_CMEQ_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
4973 if (RetVT.SimpleTy != MVT::v16i8)
4974 return 0;
4975 if ((Subtarget->hasNEON())) {
4976 return fastEmitInst_rr(AArch64::CMEQv16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4977 }
4978 return 0;
4979}
4980
4981unsigned fastEmit_AArch64ISD_CMEQ_MVT_v4i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
4982 if (RetVT.SimpleTy != MVT::v4i16)
4983 return 0;
4984 if ((Subtarget->hasNEON())) {
4985 return fastEmitInst_rr(AArch64::CMEQv4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4986 }
4987 return 0;
4988}
4989
4990unsigned fastEmit_AArch64ISD_CMEQ_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
4991 if (RetVT.SimpleTy != MVT::v8i16)
4992 return 0;
4993 if ((Subtarget->hasNEON())) {
4994 return fastEmitInst_rr(AArch64::CMEQv8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4995 }
4996 return 0;
4997}
4998
4999unsigned fastEmit_AArch64ISD_CMEQ_MVT_v2i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5000 if (RetVT.SimpleTy != MVT::v2i32)
5001 return 0;
5002 if ((Subtarget->hasNEON())) {
5003 return fastEmitInst_rr(AArch64::CMEQv2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5004 }
5005 return 0;
5006}
5007
5008unsigned fastEmit_AArch64ISD_CMEQ_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5009 if (RetVT.SimpleTy != MVT::v4i32)
5010 return 0;
5011 if ((Subtarget->hasNEON())) {
5012 return fastEmitInst_rr(AArch64::CMEQv4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5013 }
5014 return 0;
5015}
5016
5017unsigned fastEmit_AArch64ISD_CMEQ_MVT_v1i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5018 if (RetVT.SimpleTy != MVT::v1i64)
5019 return 0;
5020 if ((Subtarget->hasNEON())) {
5021 return fastEmitInst_rr(AArch64::CMEQv1i64, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5022 }
5023 return 0;
5024}
5025
5026unsigned fastEmit_AArch64ISD_CMEQ_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5027 if (RetVT.SimpleTy != MVT::v2i64)
5028 return 0;
5029 if ((Subtarget->hasNEON())) {
5030 return fastEmitInst_rr(AArch64::CMEQv2i64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5031 }
5032 return 0;
5033}
5034
5035unsigned fastEmit_AArch64ISD_CMEQ_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5036 switch (VT.SimpleTy) {
5037 case MVT::v8i8: return fastEmit_AArch64ISD_CMEQ_MVT_v8i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5038 case MVT::v16i8: return fastEmit_AArch64ISD_CMEQ_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5039 case MVT::v4i16: return fastEmit_AArch64ISD_CMEQ_MVT_v4i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5040 case MVT::v8i16: return fastEmit_AArch64ISD_CMEQ_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5041 case MVT::v2i32: return fastEmit_AArch64ISD_CMEQ_MVT_v2i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5042 case MVT::v4i32: return fastEmit_AArch64ISD_CMEQ_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5043 case MVT::v1i64: return fastEmit_AArch64ISD_CMEQ_MVT_v1i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5044 case MVT::v2i64: return fastEmit_AArch64ISD_CMEQ_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5045 default: return 0;
5046 }
5047}
5048
5049// FastEmit functions for AArch64ISD::CMGE.
5050
5051unsigned fastEmit_AArch64ISD_CMGE_MVT_v8i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5052 if (RetVT.SimpleTy != MVT::v8i8)
5053 return 0;
5054 if ((Subtarget->hasNEON())) {
5055 return fastEmitInst_rr(AArch64::CMGEv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5056 }
5057 return 0;
5058}
5059
5060unsigned fastEmit_AArch64ISD_CMGE_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5061 if (RetVT.SimpleTy != MVT::v16i8)
5062 return 0;
5063 if ((Subtarget->hasNEON())) {
5064 return fastEmitInst_rr(AArch64::CMGEv16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5065 }
5066 return 0;
5067}
5068
5069unsigned fastEmit_AArch64ISD_CMGE_MVT_v4i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5070 if (RetVT.SimpleTy != MVT::v4i16)
5071 return 0;
5072 if ((Subtarget->hasNEON())) {
5073 return fastEmitInst_rr(AArch64::CMGEv4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5074 }
5075 return 0;
5076}
5077
5078unsigned fastEmit_AArch64ISD_CMGE_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5079 if (RetVT.SimpleTy != MVT::v8i16)
5080 return 0;
5081 if ((Subtarget->hasNEON())) {
5082 return fastEmitInst_rr(AArch64::CMGEv8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5083 }
5084 return 0;
5085}
5086
5087unsigned fastEmit_AArch64ISD_CMGE_MVT_v2i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5088 if (RetVT.SimpleTy != MVT::v2i32)
5089 return 0;
5090 if ((Subtarget->hasNEON())) {
5091 return fastEmitInst_rr(AArch64::CMGEv2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5092 }
5093 return 0;
5094}
5095
5096unsigned fastEmit_AArch64ISD_CMGE_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5097 if (RetVT.SimpleTy != MVT::v4i32)
5098 return 0;
5099 if ((Subtarget->hasNEON())) {
5100 return fastEmitInst_rr(AArch64::CMGEv4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5101 }
5102 return 0;
5103}
5104
5105unsigned fastEmit_AArch64ISD_CMGE_MVT_v1i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5106 if (RetVT.SimpleTy != MVT::v1i64)
5107 return 0;
5108 if ((Subtarget->hasNEON())) {
5109 return fastEmitInst_rr(AArch64::CMGEv1i64, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5110 }
5111 return 0;
5112}
5113
5114unsigned fastEmit_AArch64ISD_CMGE_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5115 if (RetVT.SimpleTy != MVT::v2i64)
5116 return 0;
5117 if ((Subtarget->hasNEON())) {
5118 return fastEmitInst_rr(AArch64::CMGEv2i64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5119 }
5120 return 0;
5121}
5122
5123unsigned fastEmit_AArch64ISD_CMGE_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5124 switch (VT.SimpleTy) {
5125 case MVT::v8i8: return fastEmit_AArch64ISD_CMGE_MVT_v8i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5126 case MVT::v16i8: return fastEmit_AArch64ISD_CMGE_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5127 case MVT::v4i16: return fastEmit_AArch64ISD_CMGE_MVT_v4i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5128 case MVT::v8i16: return fastEmit_AArch64ISD_CMGE_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5129 case MVT::v2i32: return fastEmit_AArch64ISD_CMGE_MVT_v2i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5130 case MVT::v4i32: return fastEmit_AArch64ISD_CMGE_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5131 case MVT::v1i64: return fastEmit_AArch64ISD_CMGE_MVT_v1i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5132 case MVT::v2i64: return fastEmit_AArch64ISD_CMGE_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5133 default: return 0;
5134 }
5135}
5136
5137// FastEmit functions for AArch64ISD::CMGT.
5138
5139unsigned fastEmit_AArch64ISD_CMGT_MVT_v8i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5140 if (RetVT.SimpleTy != MVT::v8i8)
5141 return 0;
5142 if ((Subtarget->hasNEON())) {
5143 return fastEmitInst_rr(AArch64::CMGTv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5144 }
5145 return 0;
5146}
5147
5148unsigned fastEmit_AArch64ISD_CMGT_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5149 if (RetVT.SimpleTy != MVT::v16i8)
5150 return 0;
5151 if ((Subtarget->hasNEON())) {
5152 return fastEmitInst_rr(AArch64::CMGTv16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5153 }
5154 return 0;
5155}
5156
5157unsigned fastEmit_AArch64ISD_CMGT_MVT_v4i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5158 if (RetVT.SimpleTy != MVT::v4i16)
5159 return 0;
5160 if ((Subtarget->hasNEON())) {
5161 return fastEmitInst_rr(AArch64::CMGTv4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5162 }
5163 return 0;
5164}
5165
5166unsigned fastEmit_AArch64ISD_CMGT_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5167 if (RetVT.SimpleTy != MVT::v8i16)
5168 return 0;
5169 if ((Subtarget->hasNEON())) {
5170 return fastEmitInst_rr(AArch64::CMGTv8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5171 }
5172 return 0;
5173}
5174
5175unsigned fastEmit_AArch64ISD_CMGT_MVT_v2i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5176 if (RetVT.SimpleTy != MVT::v2i32)
5177 return 0;
5178 if ((Subtarget->hasNEON())) {
5179 return fastEmitInst_rr(AArch64::CMGTv2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5180 }
5181 return 0;
5182}
5183
5184unsigned fastEmit_AArch64ISD_CMGT_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5185 if (RetVT.SimpleTy != MVT::v4i32)
5186 return 0;
5187 if ((Subtarget->hasNEON())) {
5188 return fastEmitInst_rr(AArch64::CMGTv4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5189 }
5190 return 0;
5191}
5192
5193unsigned fastEmit_AArch64ISD_CMGT_MVT_v1i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5194 if (RetVT.SimpleTy != MVT::v1i64)
5195 return 0;
5196 if ((Subtarget->hasNEON())) {
5197 return fastEmitInst_rr(AArch64::CMGTv1i64, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5198 }
5199 return 0;
5200}
5201
5202unsigned fastEmit_AArch64ISD_CMGT_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5203 if (RetVT.SimpleTy != MVT::v2i64)
5204 return 0;
5205 if ((Subtarget->hasNEON())) {
5206 return fastEmitInst_rr(AArch64::CMGTv2i64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5207 }
5208 return 0;
5209}
5210
5211unsigned fastEmit_AArch64ISD_CMGT_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5212 switch (VT.SimpleTy) {
5213 case MVT::v8i8: return fastEmit_AArch64ISD_CMGT_MVT_v8i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5214 case MVT::v16i8: return fastEmit_AArch64ISD_CMGT_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5215 case MVT::v4i16: return fastEmit_AArch64ISD_CMGT_MVT_v4i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5216 case MVT::v8i16: return fastEmit_AArch64ISD_CMGT_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5217 case MVT::v2i32: return fastEmit_AArch64ISD_CMGT_MVT_v2i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5218 case MVT::v4i32: return fastEmit_AArch64ISD_CMGT_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5219 case MVT::v1i64: return fastEmit_AArch64ISD_CMGT_MVT_v1i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5220 case MVT::v2i64: return fastEmit_AArch64ISD_CMGT_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5221 default: return 0;
5222 }
5223}
5224
5225// FastEmit functions for AArch64ISD::CMHI.
5226
5227unsigned fastEmit_AArch64ISD_CMHI_MVT_v8i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5228 if (RetVT.SimpleTy != MVT::v8i8)
5229 return 0;
5230 if ((Subtarget->hasNEON())) {
5231 return fastEmitInst_rr(AArch64::CMHIv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5232 }
5233 return 0;
5234}
5235
5236unsigned fastEmit_AArch64ISD_CMHI_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5237 if (RetVT.SimpleTy != MVT::v16i8)
5238 return 0;
5239 if ((Subtarget->hasNEON())) {
5240 return fastEmitInst_rr(AArch64::CMHIv16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5241 }
5242 return 0;
5243}
5244
5245unsigned fastEmit_AArch64ISD_CMHI_MVT_v4i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5246 if (RetVT.SimpleTy != MVT::v4i16)
5247 return 0;
5248 if ((Subtarget->hasNEON())) {
5249 return fastEmitInst_rr(AArch64::CMHIv4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5250 }
5251 return 0;
5252}
5253
5254unsigned fastEmit_AArch64ISD_CMHI_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5255 if (RetVT.SimpleTy != MVT::v8i16)
5256 return 0;
5257 if ((Subtarget->hasNEON())) {
5258 return fastEmitInst_rr(AArch64::CMHIv8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5259 }
5260 return 0;
5261}
5262
5263unsigned fastEmit_AArch64ISD_CMHI_MVT_v2i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5264 if (RetVT.SimpleTy != MVT::v2i32)
5265 return 0;
5266 if ((Subtarget->hasNEON())) {
5267 return fastEmitInst_rr(AArch64::CMHIv2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5268 }
5269 return 0;
5270}
5271
5272unsigned fastEmit_AArch64ISD_CMHI_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5273 if (RetVT.SimpleTy != MVT::v4i32)
5274 return 0;
5275 if ((Subtarget->hasNEON())) {
5276 return fastEmitInst_rr(AArch64::CMHIv4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5277 }
5278 return 0;
5279}
5280
5281unsigned fastEmit_AArch64ISD_CMHI_MVT_v1i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5282 if (RetVT.SimpleTy != MVT::v1i64)
5283 return 0;
5284 if ((Subtarget->hasNEON())) {
5285 return fastEmitInst_rr(AArch64::CMHIv1i64, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5286 }
5287 return 0;
5288}
5289
5290unsigned fastEmit_AArch64ISD_CMHI_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5291 if (RetVT.SimpleTy != MVT::v2i64)
5292 return 0;
5293 if ((Subtarget->hasNEON())) {
5294 return fastEmitInst_rr(AArch64::CMHIv2i64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5295 }
5296 return 0;
5297}
5298
5299unsigned fastEmit_AArch64ISD_CMHI_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5300 switch (VT.SimpleTy) {
5301 case MVT::v8i8: return fastEmit_AArch64ISD_CMHI_MVT_v8i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5302 case MVT::v16i8: return fastEmit_AArch64ISD_CMHI_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5303 case MVT::v4i16: return fastEmit_AArch64ISD_CMHI_MVT_v4i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5304 case MVT::v8i16: return fastEmit_AArch64ISD_CMHI_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5305 case MVT::v2i32: return fastEmit_AArch64ISD_CMHI_MVT_v2i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5306 case MVT::v4i32: return fastEmit_AArch64ISD_CMHI_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5307 case MVT::v1i64: return fastEmit_AArch64ISD_CMHI_MVT_v1i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5308 case MVT::v2i64: return fastEmit_AArch64ISD_CMHI_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5309 default: return 0;
5310 }
5311}
5312
5313// FastEmit functions for AArch64ISD::CMHS.
5314
5315unsigned fastEmit_AArch64ISD_CMHS_MVT_v8i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5316 if (RetVT.SimpleTy != MVT::v8i8)
5317 return 0;
5318 if ((Subtarget->hasNEON())) {
5319 return fastEmitInst_rr(AArch64::CMHSv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5320 }
5321 return 0;
5322}
5323
5324unsigned fastEmit_AArch64ISD_CMHS_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5325 if (RetVT.SimpleTy != MVT::v16i8)
5326 return 0;
5327 if ((Subtarget->hasNEON())) {
5328 return fastEmitInst_rr(AArch64::CMHSv16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5329 }
5330 return 0;
5331}
5332
5333unsigned fastEmit_AArch64ISD_CMHS_MVT_v4i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5334 if (RetVT.SimpleTy != MVT::v4i16)
5335 return 0;
5336 if ((Subtarget->hasNEON())) {
5337 return fastEmitInst_rr(AArch64::CMHSv4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5338 }
5339 return 0;
5340}
5341
5342unsigned fastEmit_AArch64ISD_CMHS_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5343 if (RetVT.SimpleTy != MVT::v8i16)
5344 return 0;
5345 if ((Subtarget->hasNEON())) {
5346 return fastEmitInst_rr(AArch64::CMHSv8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5347 }
5348 return 0;
5349}
5350
5351unsigned fastEmit_AArch64ISD_CMHS_MVT_v2i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5352 if (RetVT.SimpleTy != MVT::v2i32)
5353 return 0;
5354 if ((Subtarget->hasNEON())) {
5355 return fastEmitInst_rr(AArch64::CMHSv2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5356 }
5357 return 0;
5358}
5359
5360unsigned fastEmit_AArch64ISD_CMHS_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5361 if (RetVT.SimpleTy != MVT::v4i32)
5362 return 0;
5363 if ((Subtarget->hasNEON())) {
5364 return fastEmitInst_rr(AArch64::CMHSv4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5365 }
5366 return 0;
5367}
5368
5369unsigned fastEmit_AArch64ISD_CMHS_MVT_v1i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5370 if (RetVT.SimpleTy != MVT::v1i64)
5371 return 0;
5372 if ((Subtarget->hasNEON())) {
5373 return fastEmitInst_rr(AArch64::CMHSv1i64, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5374 }
5375 return 0;
5376}
5377
5378unsigned fastEmit_AArch64ISD_CMHS_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5379 if (RetVT.SimpleTy != MVT::v2i64)
5380 return 0;
5381 if ((Subtarget->hasNEON())) {
5382 return fastEmitInst_rr(AArch64::CMHSv2i64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5383 }
5384 return 0;
5385}
5386
5387unsigned fastEmit_AArch64ISD_CMHS_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5388 switch (VT.SimpleTy) {
5389 case MVT::v8i8: return fastEmit_AArch64ISD_CMHS_MVT_v8i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5390 case MVT::v16i8: return fastEmit_AArch64ISD_CMHS_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5391 case MVT::v4i16: return fastEmit_AArch64ISD_CMHS_MVT_v4i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5392 case MVT::v8i16: return fastEmit_AArch64ISD_CMHS_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5393 case MVT::v2i32: return fastEmit_AArch64ISD_CMHS_MVT_v2i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5394 case MVT::v4i32: return fastEmit_AArch64ISD_CMHS_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5395 case MVT::v1i64: return fastEmit_AArch64ISD_CMHS_MVT_v1i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5396 case MVT::v2i64: return fastEmit_AArch64ISD_CMHS_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5397 default: return 0;
5398 }
5399}
5400
5401// FastEmit functions for AArch64ISD::FCMEQ.
5402
5403unsigned fastEmit_AArch64ISD_FCMEQ_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5404 if (RetVT.SimpleTy != MVT::i32)
5405 return 0;
5406 if ((Subtarget->hasNEON())) {
5407 return fastEmitInst_rr(AArch64::FCMEQ32, &AArch64::FPR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5408 }
5409 return 0;
5410}
5411
5412unsigned fastEmit_AArch64ISD_FCMEQ_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5413 if (RetVT.SimpleTy != MVT::i64)
5414 return 0;
5415 if ((Subtarget->hasNEON())) {
5416 return fastEmitInst_rr(AArch64::FCMEQ64, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5417 }
5418 return 0;
5419}
5420
5421unsigned fastEmit_AArch64ISD_FCMEQ_MVT_v4f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5422 if (RetVT.SimpleTy != MVT::v4i16)
5423 return 0;
5424 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
5425 return fastEmitInst_rr(AArch64::FCMEQv4f16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5426 }
5427 return 0;
5428}
5429
5430unsigned fastEmit_AArch64ISD_FCMEQ_MVT_v8f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5431 if (RetVT.SimpleTy != MVT::v8i16)
5432 return 0;
5433 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
5434 return fastEmitInst_rr(AArch64::FCMEQv8f16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5435 }
5436 return 0;
5437}
5438
5439unsigned fastEmit_AArch64ISD_FCMEQ_MVT_v2f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5440 if (RetVT.SimpleTy != MVT::v2i32)
5441 return 0;
5442 if ((Subtarget->hasNEON())) {
5443 return fastEmitInst_rr(AArch64::FCMEQv2f32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5444 }
5445 return 0;
5446}
5447
5448unsigned fastEmit_AArch64ISD_FCMEQ_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5449 if (RetVT.SimpleTy != MVT::v4i32)
5450 return 0;
5451 if ((Subtarget->hasNEON())) {
5452 return fastEmitInst_rr(AArch64::FCMEQv4f32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5453 }
5454 return 0;
5455}
5456
5457unsigned fastEmit_AArch64ISD_FCMEQ_MVT_v1f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5458 if (RetVT.SimpleTy != MVT::v1i64)
5459 return 0;
5460 if ((Subtarget->hasNEON())) {
5461 return fastEmitInst_rr(AArch64::FCMEQ64, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5462 }
5463 return 0;
5464}
5465
5466unsigned fastEmit_AArch64ISD_FCMEQ_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5467 if (RetVT.SimpleTy != MVT::v2i64)
5468 return 0;
5469 if ((Subtarget->hasNEON())) {
5470 return fastEmitInst_rr(AArch64::FCMEQv2f64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5471 }
5472 return 0;
5473}
5474
5475unsigned fastEmit_AArch64ISD_FCMEQ_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5476 switch (VT.SimpleTy) {
5477 case MVT::f32: return fastEmit_AArch64ISD_FCMEQ_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5478 case MVT::f64: return fastEmit_AArch64ISD_FCMEQ_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5479 case MVT::v4f16: return fastEmit_AArch64ISD_FCMEQ_MVT_v4f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5480 case MVT::v8f16: return fastEmit_AArch64ISD_FCMEQ_MVT_v8f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5481 case MVT::v2f32: return fastEmit_AArch64ISD_FCMEQ_MVT_v2f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5482 case MVT::v4f32: return fastEmit_AArch64ISD_FCMEQ_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5483 case MVT::v1f64: return fastEmit_AArch64ISD_FCMEQ_MVT_v1f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5484 case MVT::v2f64: return fastEmit_AArch64ISD_FCMEQ_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5485 default: return 0;
5486 }
5487}
5488
5489// FastEmit functions for AArch64ISD::FCMGE.
5490
5491unsigned fastEmit_AArch64ISD_FCMGE_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5492 if (RetVT.SimpleTy != MVT::i32)
5493 return 0;
5494 if ((Subtarget->hasNEON())) {
5495 return fastEmitInst_rr(AArch64::FCMGE32, &AArch64::FPR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5496 }
5497 return 0;
5498}
5499
5500unsigned fastEmit_AArch64ISD_FCMGE_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5501 if (RetVT.SimpleTy != MVT::i64)
5502 return 0;
5503 if ((Subtarget->hasNEON())) {
5504 return fastEmitInst_rr(AArch64::FCMGE64, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5505 }
5506 return 0;
5507}
5508
5509unsigned fastEmit_AArch64ISD_FCMGE_MVT_v4f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5510 if (RetVT.SimpleTy != MVT::v4i16)
5511 return 0;
5512 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
5513 return fastEmitInst_rr(AArch64::FCMGEv4f16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5514 }
5515 return 0;
5516}
5517
5518unsigned fastEmit_AArch64ISD_FCMGE_MVT_v8f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5519 if (RetVT.SimpleTy != MVT::v8i16)
5520 return 0;
5521 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
5522 return fastEmitInst_rr(AArch64::FCMGEv8f16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5523 }
5524 return 0;
5525}
5526
5527unsigned fastEmit_AArch64ISD_FCMGE_MVT_v2f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5528 if (RetVT.SimpleTy != MVT::v2i32)
5529 return 0;
5530 if ((Subtarget->hasNEON())) {
5531 return fastEmitInst_rr(AArch64::FCMGEv2f32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5532 }
5533 return 0;
5534}
5535
5536unsigned fastEmit_AArch64ISD_FCMGE_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5537 if (RetVT.SimpleTy != MVT::v4i32)
5538 return 0;
5539 if ((Subtarget->hasNEON())) {
5540 return fastEmitInst_rr(AArch64::FCMGEv4f32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5541 }
5542 return 0;
5543}
5544
5545unsigned fastEmit_AArch64ISD_FCMGE_MVT_v1f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5546 if (RetVT.SimpleTy != MVT::v1i64)
5547 return 0;
5548 if ((Subtarget->hasNEON())) {
5549 return fastEmitInst_rr(AArch64::FCMGE64, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5550 }
5551 return 0;
5552}
5553
5554unsigned fastEmit_AArch64ISD_FCMGE_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5555 if (RetVT.SimpleTy != MVT::v2i64)
5556 return 0;
5557 if ((Subtarget->hasNEON())) {
5558 return fastEmitInst_rr(AArch64::FCMGEv2f64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5559 }
5560 return 0;
5561}
5562
5563unsigned fastEmit_AArch64ISD_FCMGE_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5564 switch (VT.SimpleTy) {
5565 case MVT::f32: return fastEmit_AArch64ISD_FCMGE_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5566 case MVT::f64: return fastEmit_AArch64ISD_FCMGE_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5567 case MVT::v4f16: return fastEmit_AArch64ISD_FCMGE_MVT_v4f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5568 case MVT::v8f16: return fastEmit_AArch64ISD_FCMGE_MVT_v8f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5569 case MVT::v2f32: return fastEmit_AArch64ISD_FCMGE_MVT_v2f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5570 case MVT::v4f32: return fastEmit_AArch64ISD_FCMGE_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5571 case MVT::v1f64: return fastEmit_AArch64ISD_FCMGE_MVT_v1f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5572 case MVT::v2f64: return fastEmit_AArch64ISD_FCMGE_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5573 default: return 0;
5574 }
5575}
5576
5577// FastEmit functions for AArch64ISD::FCMGT.
5578
5579unsigned fastEmit_AArch64ISD_FCMGT_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5580 if (RetVT.SimpleTy != MVT::i32)
5581 return 0;
5582 if ((Subtarget->hasNEON())) {
5583 return fastEmitInst_rr(AArch64::FCMGT32, &AArch64::FPR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5584 }
5585 return 0;
5586}
5587
5588unsigned fastEmit_AArch64ISD_FCMGT_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5589 if (RetVT.SimpleTy != MVT::i64)
5590 return 0;
5591 if ((Subtarget->hasNEON())) {
5592 return fastEmitInst_rr(AArch64::FCMGT64, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5593 }
5594 return 0;
5595}
5596
5597unsigned fastEmit_AArch64ISD_FCMGT_MVT_v4f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5598 if (RetVT.SimpleTy != MVT::v4i16)
5599 return 0;
5600 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
5601 return fastEmitInst_rr(AArch64::FCMGTv4f16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5602 }
5603 return 0;
5604}
5605
5606unsigned fastEmit_AArch64ISD_FCMGT_MVT_v8f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5607 if (RetVT.SimpleTy != MVT::v8i16)
5608 return 0;
5609 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
5610 return fastEmitInst_rr(AArch64::FCMGTv8f16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5611 }
5612 return 0;
5613}
5614
5615unsigned fastEmit_AArch64ISD_FCMGT_MVT_v2f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5616 if (RetVT.SimpleTy != MVT::v2i32)
5617 return 0;
5618 if ((Subtarget->hasNEON())) {
5619 return fastEmitInst_rr(AArch64::FCMGTv2f32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5620 }
5621 return 0;
5622}
5623
5624unsigned fastEmit_AArch64ISD_FCMGT_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5625 if (RetVT.SimpleTy != MVT::v4i32)
5626 return 0;
5627 if ((Subtarget->hasNEON())) {
5628 return fastEmitInst_rr(AArch64::FCMGTv4f32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5629 }
5630 return 0;
5631}
5632
5633unsigned fastEmit_AArch64ISD_FCMGT_MVT_v1f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5634 if (RetVT.SimpleTy != MVT::v1i64)
5635 return 0;
5636 if ((Subtarget->hasNEON())) {
5637 return fastEmitInst_rr(AArch64::FCMGT64, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5638 }
5639 return 0;
5640}
5641
5642unsigned fastEmit_AArch64ISD_FCMGT_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5643 if (RetVT.SimpleTy != MVT::v2i64)
5644 return 0;
5645 if ((Subtarget->hasNEON())) {
5646 return fastEmitInst_rr(AArch64::FCMGTv2f64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5647 }
5648 return 0;
5649}
5650
5651unsigned fastEmit_AArch64ISD_FCMGT_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5652 switch (VT.SimpleTy) {
5653 case MVT::f32: return fastEmit_AArch64ISD_FCMGT_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5654 case MVT::f64: return fastEmit_AArch64ISD_FCMGT_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5655 case MVT::v4f16: return fastEmit_AArch64ISD_FCMGT_MVT_v4f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5656 case MVT::v8f16: return fastEmit_AArch64ISD_FCMGT_MVT_v8f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5657 case MVT::v2f32: return fastEmit_AArch64ISD_FCMGT_MVT_v2f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5658 case MVT::v4f32: return fastEmit_AArch64ISD_FCMGT_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5659 case MVT::v1f64: return fastEmit_AArch64ISD_FCMGT_MVT_v1f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5660 case MVT::v2f64: return fastEmit_AArch64ISD_FCMGT_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5661 default: return 0;
5662 }
5663}
5664
5665// FastEmit functions for AArch64ISD::FCMP.
5666
5667unsigned fastEmit_AArch64ISD_FCMP_MVT_f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5668 if (RetVT.SimpleTy != MVT::isVoid)
5669 return 0;
5670 if ((Subtarget->hasFullFP16())) {
5671 return fastEmitInst_rr(AArch64::FCMPHrr, &AArch64::FPR16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5672 }
5673 return 0;
5674}
5675
5676unsigned fastEmit_AArch64ISD_FCMP_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5677 if (RetVT.SimpleTy != MVT::isVoid)
5678 return 0;
5679 if ((Subtarget->hasFPARMv8())) {
5680 return fastEmitInst_rr(AArch64::FCMPSrr, &AArch64::FPR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5681 }
5682 return 0;
5683}
5684
5685unsigned fastEmit_AArch64ISD_FCMP_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5686 if (RetVT.SimpleTy != MVT::isVoid)
5687 return 0;
5688 if ((Subtarget->hasFPARMv8())) {
5689 return fastEmitInst_rr(AArch64::FCMPDrr, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5690 }
5691 return 0;
5692}
5693
5694unsigned fastEmit_AArch64ISD_FCMP_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5695 switch (VT.SimpleTy) {
5696 case MVT::f16: return fastEmit_AArch64ISD_FCMP_MVT_f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5697 case MVT::f32: return fastEmit_AArch64ISD_FCMP_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5698 case MVT::f64: return fastEmit_AArch64ISD_FCMP_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5699 default: return 0;
5700 }
5701}
5702
5703// FastEmit functions for AArch64ISD::FRECPS.
5704
5705unsigned fastEmit_AArch64ISD_FRECPS_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5706 if (RetVT.SimpleTy != MVT::f32)
5707 return 0;
5708 return fastEmitInst_rr(AArch64::FRECPS32, &AArch64::FPR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5709}
5710
5711unsigned fastEmit_AArch64ISD_FRECPS_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5712 if (RetVT.SimpleTy != MVT::f64)
5713 return 0;
5714 return fastEmitInst_rr(AArch64::FRECPS64, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5715}
5716
5717unsigned fastEmit_AArch64ISD_FRECPS_MVT_v2f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5718 if (RetVT.SimpleTy != MVT::v2f32)
5719 return 0;
5720 return fastEmitInst_rr(AArch64::FRECPSv2f32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5721}
5722
5723unsigned fastEmit_AArch64ISD_FRECPS_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5724 if (RetVT.SimpleTy != MVT::v4f32)
5725 return 0;
5726 return fastEmitInst_rr(AArch64::FRECPSv4f32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5727}
5728
5729unsigned fastEmit_AArch64ISD_FRECPS_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5730 if (RetVT.SimpleTy != MVT::v2f64)
5731 return 0;
5732 return fastEmitInst_rr(AArch64::FRECPSv2f64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5733}
5734
5735unsigned fastEmit_AArch64ISD_FRECPS_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5736 switch (VT.SimpleTy) {
5737 case MVT::f32: return fastEmit_AArch64ISD_FRECPS_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5738 case MVT::f64: return fastEmit_AArch64ISD_FRECPS_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5739 case MVT::v2f32: return fastEmit_AArch64ISD_FRECPS_MVT_v2f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5740 case MVT::v4f32: return fastEmit_AArch64ISD_FRECPS_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5741 case MVT::v2f64: return fastEmit_AArch64ISD_FRECPS_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5742 default: return 0;
5743 }
5744}
5745
5746// FastEmit functions for AArch64ISD::FRSQRTS.
5747
5748unsigned fastEmit_AArch64ISD_FRSQRTS_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5749 if (RetVT.SimpleTy != MVT::f32)
5750 return 0;
5751 return fastEmitInst_rr(AArch64::FRSQRTS32, &AArch64::FPR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5752}
5753
5754unsigned fastEmit_AArch64ISD_FRSQRTS_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5755 if (RetVT.SimpleTy != MVT::f64)
5756 return 0;
5757 return fastEmitInst_rr(AArch64::FRSQRTS64, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5758}
5759
5760unsigned fastEmit_AArch64ISD_FRSQRTS_MVT_v2f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5761 if (RetVT.SimpleTy != MVT::v2f32)
5762 return 0;
5763 return fastEmitInst_rr(AArch64::FRSQRTSv2f32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5764}
5765
5766unsigned fastEmit_AArch64ISD_FRSQRTS_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5767 if (RetVT.SimpleTy != MVT::v4f32)
5768 return 0;
5769 return fastEmitInst_rr(AArch64::FRSQRTSv4f32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5770}
5771
5772unsigned fastEmit_AArch64ISD_FRSQRTS_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5773 if (RetVT.SimpleTy != MVT::v2f64)
5774 return 0;
5775 return fastEmitInst_rr(AArch64::FRSQRTSv2f64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5776}
5777
5778unsigned fastEmit_AArch64ISD_FRSQRTS_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5779 switch (VT.SimpleTy) {
5780 case MVT::f32: return fastEmit_AArch64ISD_FRSQRTS_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5781 case MVT::f64: return fastEmit_AArch64ISD_FRSQRTS_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5782 case MVT::v2f32: return fastEmit_AArch64ISD_FRSQRTS_MVT_v2f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5783 case MVT::v4f32: return fastEmit_AArch64ISD_FRSQRTS_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5784 case MVT::v2f64: return fastEmit_AArch64ISD_FRSQRTS_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5785 default: return 0;
5786 }
5787}
5788
5789// FastEmit functions for AArch64ISD::INDEX_VECTOR.
5790
5791unsigned fastEmit_AArch64ISD_INDEX_VECTOR_MVT_i32_MVT_nxv16i8_rr(unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5792 if ((Subtarget->hasSVE())) {
5793 return fastEmitInst_rr(AArch64::INDEX_RR_B, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5794 }
5795 return 0;
5796}
5797
5798unsigned fastEmit_AArch64ISD_INDEX_VECTOR_MVT_i32_MVT_nxv8i16_rr(unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5799 if ((Subtarget->hasSVE())) {
5800 return fastEmitInst_rr(AArch64::INDEX_RR_H, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5801 }
5802 return 0;
5803}
5804
5805unsigned fastEmit_AArch64ISD_INDEX_VECTOR_MVT_i32_MVT_nxv4i32_rr(unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5806 if ((Subtarget->hasSVE())) {
5807 return fastEmitInst_rr(AArch64::INDEX_RR_S, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5808 }
5809 return 0;
5810}
5811
5812unsigned fastEmit_AArch64ISD_INDEX_VECTOR_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5813switch (RetVT.SimpleTy) {
5814 case MVT::nxv16i8: return fastEmit_AArch64ISD_INDEX_VECTOR_MVT_i32_MVT_nxv16i8_rr(Op0, Op0IsKill, Op1, Op1IsKill);
5815 case MVT::nxv8i16: return fastEmit_AArch64ISD_INDEX_VECTOR_MVT_i32_MVT_nxv8i16_rr(Op0, Op0IsKill, Op1, Op1IsKill);
5816 case MVT::nxv4i32: return fastEmit_AArch64ISD_INDEX_VECTOR_MVT_i32_MVT_nxv4i32_rr(Op0, Op0IsKill, Op1, Op1IsKill);
5817 default: return 0;
5818}
5819}
5820
5821unsigned fastEmit_AArch64ISD_INDEX_VECTOR_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5822 if (RetVT.SimpleTy != MVT::nxv2i64)
5823 return 0;
5824 if ((Subtarget->hasSVE())) {
5825 return fastEmitInst_rr(AArch64::INDEX_RR_D, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5826 }
5827 return 0;
5828}
5829
5830unsigned fastEmit_AArch64ISD_INDEX_VECTOR_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5831 switch (VT.SimpleTy) {
5832 case MVT::i32: return fastEmit_AArch64ISD_INDEX_VECTOR_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5833 case MVT::i64: return fastEmit_AArch64ISD_INDEX_VECTOR_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5834 default: return 0;
5835 }
5836}
5837
5838// FastEmit functions for AArch64ISD::PTEST.
5839
5840unsigned fastEmit_AArch64ISD_PTEST_MVT_nxv2i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5841 if (RetVT.SimpleTy != MVT::isVoid)
5842 return 0;
5843 if ((Subtarget->hasSVE())) {
5844 return fastEmitInst_rr(AArch64::PTEST_PP, &AArch64::PPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5845 }
5846 return 0;
5847}
5848
5849unsigned fastEmit_AArch64ISD_PTEST_MVT_nxv4i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5850 if (RetVT.SimpleTy != MVT::isVoid)
5851 return 0;
5852 if ((Subtarget->hasSVE())) {
5853 return fastEmitInst_rr(AArch64::PTEST_PP, &AArch64::PPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5854 }
5855 return 0;
5856}
5857
5858unsigned fastEmit_AArch64ISD_PTEST_MVT_nxv8i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5859 if (RetVT.SimpleTy != MVT::isVoid)
5860 return 0;
5861 if ((Subtarget->hasSVE())) {
5862 return fastEmitInst_rr(AArch64::PTEST_PP, &AArch64::PPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5863 }
5864 return 0;
5865}
5866
5867unsigned fastEmit_AArch64ISD_PTEST_MVT_nxv16i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5868 if (RetVT.SimpleTy != MVT::isVoid)
5869 return 0;
5870 if ((Subtarget->hasSVE())) {
5871 return fastEmitInst_rr(AArch64::PTEST_PP, &AArch64::PPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5872 }
5873 return 0;
5874}
5875
5876unsigned fastEmit_AArch64ISD_PTEST_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5877 switch (VT.SimpleTy) {
5878 case MVT::nxv2i1: return fastEmit_AArch64ISD_PTEST_MVT_nxv2i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5879 case MVT::nxv4i1: return fastEmit_AArch64ISD_PTEST_MVT_nxv4i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5880 case MVT::nxv8i1: return fastEmit_AArch64ISD_PTEST_MVT_nxv8i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5881 case MVT::nxv16i1: return fastEmit_AArch64ISD_PTEST_MVT_nxv16i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5882 default: return 0;
5883 }
5884}
5885
5886// FastEmit functions for AArch64ISD::SABD.
5887
5888unsigned fastEmit_AArch64ISD_SABD_MVT_v8i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5889 if (RetVT.SimpleTy != MVT::v8i8)
5890 return 0;
5891 if ((Subtarget->hasNEON())) {
5892 return fastEmitInst_rr(AArch64::SABDv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5893 }
5894 return 0;
5895}
5896
5897unsigned fastEmit_AArch64ISD_SABD_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5898 if (RetVT.SimpleTy != MVT::v16i8)
5899 return 0;
5900 if ((Subtarget->hasNEON())) {
5901 return fastEmitInst_rr(AArch64::SABDv16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5902 }
5903 return 0;
5904}
5905
5906unsigned fastEmit_AArch64ISD_SABD_MVT_v4i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5907 if (RetVT.SimpleTy != MVT::v4i16)
5908 return 0;
5909 if ((Subtarget->hasNEON())) {
5910 return fastEmitInst_rr(AArch64::SABDv4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5911 }
5912 return 0;
5913}
5914
5915unsigned fastEmit_AArch64ISD_SABD_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5916 if (RetVT.SimpleTy != MVT::v8i16)
5917 return 0;
5918 if ((Subtarget->hasNEON())) {
5919 return fastEmitInst_rr(AArch64::SABDv8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5920 }
5921 return 0;
5922}
5923
5924unsigned fastEmit_AArch64ISD_SABD_MVT_v2i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5925 if (RetVT.SimpleTy != MVT::v2i32)
5926 return 0;
5927 if ((Subtarget->hasNEON())) {
5928 return fastEmitInst_rr(AArch64::SABDv2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5929 }
5930 return 0;
5931}
5932
5933unsigned fastEmit_AArch64ISD_SABD_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5934 if (RetVT.SimpleTy != MVT::v4i32)
5935 return 0;
5936 if ((Subtarget->hasNEON())) {
5937 return fastEmitInst_rr(AArch64::SABDv4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5938 }
5939 return 0;
5940}
5941
5942unsigned fastEmit_AArch64ISD_SABD_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5943 switch (VT.SimpleTy) {
5944 case MVT::v8i8: return fastEmit_AArch64ISD_SABD_MVT_v8i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5945 case MVT::v16i8: return fastEmit_AArch64ISD_SABD_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5946 case MVT::v4i16: return fastEmit_AArch64ISD_SABD_MVT_v4i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5947 case MVT::v8i16: return fastEmit_AArch64ISD_SABD_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5948 case MVT::v2i32: return fastEmit_AArch64ISD_SABD_MVT_v2i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5949 case MVT::v4i32: return fastEmit_AArch64ISD_SABD_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5950 default: return 0;
5951 }
5952}
5953
5954// FastEmit functions for AArch64ISD::SHADD.
5955
5956unsigned fastEmit_AArch64ISD_SHADD_MVT_v8i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5957 if (RetVT.SimpleTy != MVT::v8i8)
5958 return 0;
5959 if ((Subtarget->hasNEON())) {
5960 return fastEmitInst_rr(AArch64::SHADDv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5961 }
5962 return 0;
5963}
5964
5965unsigned fastEmit_AArch64ISD_SHADD_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5966 if (RetVT.SimpleTy != MVT::v16i8)
5967 return 0;
5968 if ((Subtarget->hasNEON())) {
5969 return fastEmitInst_rr(AArch64::SHADDv16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5970 }
5971 return 0;
5972}
5973
5974unsigned fastEmit_AArch64ISD_SHADD_MVT_v4i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5975 if (RetVT.SimpleTy != MVT::v4i16)
5976 return 0;
5977 if ((Subtarget->hasNEON())) {
5978 return fastEmitInst_rr(AArch64::SHADDv4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5979 }
5980 return 0;
5981}
5982
5983unsigned fastEmit_AArch64ISD_SHADD_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5984 if (RetVT.SimpleTy != MVT::v8i16)
5985 return 0;
5986 if ((Subtarget->hasNEON())) {
5987 return fastEmitInst_rr(AArch64::SHADDv8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5988 }
5989 return 0;
5990}
5991
5992unsigned fastEmit_AArch64ISD_SHADD_MVT_v2i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5993 if (RetVT.SimpleTy != MVT::v2i32)
5994 return 0;
5995 if ((Subtarget->hasNEON())) {
5996 return fastEmitInst_rr(AArch64::SHADDv2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5997 }
5998 return 0;
5999}
6000
6001unsigned fastEmit_AArch64ISD_SHADD_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6002 if (RetVT.SimpleTy != MVT::v4i32)
6003 return 0;
6004 if ((Subtarget->hasNEON())) {
6005 return fastEmitInst_rr(AArch64::SHADDv4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6006 }
6007 return 0;
6008}
6009
6010unsigned fastEmit_AArch64ISD_SHADD_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6011 switch (VT.SimpleTy) {
6012 case MVT::v8i8: return fastEmit_AArch64ISD_SHADD_MVT_v8i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6013 case MVT::v16i8: return fastEmit_AArch64ISD_SHADD_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6014 case MVT::v4i16: return fastEmit_AArch64ISD_SHADD_MVT_v4i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6015 case MVT::v8i16: return fastEmit_AArch64ISD_SHADD_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6016 case MVT::v2i32: return fastEmit_AArch64ISD_SHADD_MVT_v2i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6017 case MVT::v4i32: return fastEmit_AArch64ISD_SHADD_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6018 default: return 0;
6019 }
6020}
6021
6022// FastEmit functions for AArch64ISD::SMULL.
6023
6024unsigned fastEmit_AArch64ISD_SMULL_MVT_v8i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6025 if (RetVT.SimpleTy != MVT::v8i16)
6026 return 0;
6027 return fastEmitInst_rr(AArch64::SMULLv8i8_v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6028}
6029
6030unsigned fastEmit_AArch64ISD_SMULL_MVT_v4i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6031 if (RetVT.SimpleTy != MVT::v4i32)
6032 return 0;
6033 return fastEmitInst_rr(AArch64::SMULLv4i16_v4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6034}
6035
6036unsigned fastEmit_AArch64ISD_SMULL_MVT_v2i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6037 if (RetVT.SimpleTy != MVT::v2i64)
6038 return 0;
6039 return fastEmitInst_rr(AArch64::SMULLv2i32_v2i64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6040}
6041
6042unsigned fastEmit_AArch64ISD_SMULL_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6043 switch (VT.SimpleTy) {
6044 case MVT::v8i8: return fastEmit_AArch64ISD_SMULL_MVT_v8i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6045 case MVT::v4i16: return fastEmit_AArch64ISD_SMULL_MVT_v4i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6046 case MVT::v2i32: return fastEmit_AArch64ISD_SMULL_MVT_v2i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6047 default: return 0;
6048 }
6049}
6050
6051// FastEmit functions for AArch64ISD::SRHADD.
6052
6053unsigned fastEmit_AArch64ISD_SRHADD_MVT_v8i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6054 if (RetVT.SimpleTy != MVT::v8i8)
6055 return 0;
6056 if ((Subtarget->hasNEON())) {
6057 return fastEmitInst_rr(AArch64::SRHADDv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6058 }
6059 return 0;
6060}
6061
6062unsigned fastEmit_AArch64ISD_SRHADD_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6063 if (RetVT.SimpleTy != MVT::v16i8)
6064 return 0;
6065 if ((Subtarget->hasNEON())) {
6066 return fastEmitInst_rr(AArch64::SRHADDv16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6067 }
6068 return 0;
6069}
6070
6071unsigned fastEmit_AArch64ISD_SRHADD_MVT_v4i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6072 if (RetVT.SimpleTy != MVT::v4i16)
6073 return 0;
6074 if ((Subtarget->hasNEON())) {
6075 return fastEmitInst_rr(AArch64::SRHADDv4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6076 }
6077 return 0;
6078}
6079
6080unsigned fastEmit_AArch64ISD_SRHADD_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6081 if (RetVT.SimpleTy != MVT::v8i16)
6082 return 0;
6083 if ((Subtarget->hasNEON())) {
6084 return fastEmitInst_rr(AArch64::SRHADDv8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6085 }
6086 return 0;
6087}
6088
6089unsigned fastEmit_AArch64ISD_SRHADD_MVT_v2i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6090 if (RetVT.SimpleTy != MVT::v2i32)
6091 return 0;
6092 if ((Subtarget->hasNEON())) {
6093 return fastEmitInst_rr(AArch64::SRHADDv2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6094 }
6095 return 0;
6096}
6097
6098unsigned fastEmit_AArch64ISD_SRHADD_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6099 if (RetVT.SimpleTy != MVT::v4i32)
6100 return 0;
6101 if ((Subtarget->hasNEON())) {
6102 return fastEmitInst_rr(AArch64::SRHADDv4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6103 }
6104 return 0;
6105}
6106
6107unsigned fastEmit_AArch64ISD_SRHADD_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6108 switch (VT.SimpleTy) {
6109 case MVT::v8i8: return fastEmit_AArch64ISD_SRHADD_MVT_v8i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6110 case MVT::v16i8: return fastEmit_AArch64ISD_SRHADD_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6111 case MVT::v4i16: return fastEmit_AArch64ISD_SRHADD_MVT_v4i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6112 case MVT::v8i16: return fastEmit_AArch64ISD_SRHADD_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6113 case MVT::v2i32: return fastEmit_AArch64ISD_SRHADD_MVT_v2i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6114 case MVT::v4i32: return fastEmit_AArch64ISD_SRHADD_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6115 default: return 0;
6116 }
6117}
6118
6119// FastEmit functions for AArch64ISD::STRICT_FCMP.
6120
6121unsigned fastEmit_AArch64ISD_STRICT_FCMP_MVT_f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6122 if (RetVT.SimpleTy != MVT::isVoid)
6123 return 0;
6124 if ((Subtarget->hasFullFP16())) {
6125 return fastEmitInst_rr(AArch64::FCMPHrr, &AArch64::FPR16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6126 }
6127 return 0;
6128}
6129
6130unsigned fastEmit_AArch64ISD_STRICT_FCMP_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6131 if (RetVT.SimpleTy != MVT::isVoid)
6132 return 0;
6133 if ((Subtarget->hasFPARMv8())) {
6134 return fastEmitInst_rr(AArch64::FCMPSrr, &AArch64::FPR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6135 }
6136 return 0;
6137}
6138
6139unsigned fastEmit_AArch64ISD_STRICT_FCMP_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6140 if (RetVT.SimpleTy != MVT::isVoid)
6141 return 0;
6142 if ((Subtarget->hasFPARMv8())) {
6143 return fastEmitInst_rr(AArch64::FCMPDrr, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6144 }
6145 return 0;
6146}
6147
6148unsigned fastEmit_AArch64ISD_STRICT_FCMP_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6149 switch (VT.SimpleTy) {
6150 case MVT::f16: return fastEmit_AArch64ISD_STRICT_FCMP_MVT_f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6151 case MVT::f32: return fastEmit_AArch64ISD_STRICT_FCMP_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6152 case MVT::f64: return fastEmit_AArch64ISD_STRICT_FCMP_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6153 default: return 0;
6154 }
6155}
6156
6157// FastEmit functions for AArch64ISD::STRICT_FCMPE.
6158
6159unsigned fastEmit_AArch64ISD_STRICT_FCMPE_MVT_f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6160 if (RetVT.SimpleTy != MVT::isVoid)
6161 return 0;
6162 if ((Subtarget->hasFullFP16())) {
6163 return fastEmitInst_rr(AArch64::FCMPEHrr, &AArch64::FPR16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6164 }
6165 return 0;
6166}
6167
6168unsigned fastEmit_AArch64ISD_STRICT_FCMPE_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6169 if (RetVT.SimpleTy != MVT::isVoid)
6170 return 0;
6171 if ((Subtarget->hasFPARMv8())) {
6172 return fastEmitInst_rr(AArch64::FCMPESrr, &AArch64::FPR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6173 }
6174 return 0;
6175}
6176
6177unsigned fastEmit_AArch64ISD_STRICT_FCMPE_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6178 if (RetVT.SimpleTy != MVT::isVoid)
6179 return 0;
6180 if ((Subtarget->hasFPARMv8())) {
6181 return fastEmitInst_rr(AArch64::FCMPEDrr, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6182 }
6183 return 0;
6184}
6185
6186unsigned fastEmit_AArch64ISD_STRICT_FCMPE_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6187 switch (VT.SimpleTy) {
6188 case MVT::f16: return fastEmit_AArch64ISD_STRICT_FCMPE_MVT_f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6189 case MVT::f32: return fastEmit_AArch64ISD_STRICT_FCMPE_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6190 case MVT::f64: return fastEmit_AArch64ISD_STRICT_FCMPE_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6191 default: return 0;
6192 }
6193}
6194
6195// FastEmit functions for AArch64ISD::TBL.
6196
6197unsigned fastEmit_AArch64ISD_TBL_MVT_nxv16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6198 if (RetVT.SimpleTy != MVT::nxv16i8)
6199 return 0;
6200 if ((Subtarget->hasSVE())) {
6201 return fastEmitInst_rr(AArch64::TBL_ZZZ_B, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6202 }
6203 return 0;
6204}
6205
6206unsigned fastEmit_AArch64ISD_TBL_MVT_nxv8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6207 if (RetVT.SimpleTy != MVT::nxv8i16)
6208 return 0;
6209 if ((Subtarget->hasSVE())) {
6210 return fastEmitInst_rr(AArch64::TBL_ZZZ_H, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6211 }
6212 return 0;
6213}
6214
6215unsigned fastEmit_AArch64ISD_TBL_MVT_nxv4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6216 if (RetVT.SimpleTy != MVT::nxv4i32)
6217 return 0;
6218 if ((Subtarget->hasSVE())) {
6219 return fastEmitInst_rr(AArch64::TBL_ZZZ_S, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6220 }
6221 return 0;
6222}
6223
6224unsigned fastEmit_AArch64ISD_TBL_MVT_nxv2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6225 if (RetVT.SimpleTy != MVT::nxv2i64)
6226 return 0;
6227 if ((Subtarget->hasSVE())) {
6228 return fastEmitInst_rr(AArch64::TBL_ZZZ_D, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6229 }
6230 return 0;
6231}
6232
6233unsigned fastEmit_AArch64ISD_TBL_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6234 switch (VT.SimpleTy) {
6235 case MVT::nxv16i8: return fastEmit_AArch64ISD_TBL_MVT_nxv16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6236 case MVT::nxv8i16: return fastEmit_AArch64ISD_TBL_MVT_nxv8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6237 case MVT::nxv4i32: return fastEmit_AArch64ISD_TBL_MVT_nxv4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6238 case MVT::nxv2i64: return fastEmit_AArch64ISD_TBL_MVT_nxv2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6239 default: return 0;
6240 }
6241}
6242
6243// FastEmit functions for AArch64ISD::TRN1.
6244
6245unsigned fastEmit_AArch64ISD_TRN1_MVT_v8i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6246 if (RetVT.SimpleTy != MVT::v8i8)
6247 return 0;
6248 if ((Subtarget->hasNEON())) {
6249 return fastEmitInst_rr(AArch64::TRN1v8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6250 }
6251 return 0;
6252}
6253
6254unsigned fastEmit_AArch64ISD_TRN1_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6255 if (RetVT.SimpleTy != MVT::v16i8)
6256 return 0;
6257 if ((Subtarget->hasNEON())) {
6258 return fastEmitInst_rr(AArch64::TRN1v16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6259 }
6260 return 0;
6261}
6262
6263unsigned fastEmit_AArch64ISD_TRN1_MVT_v4i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6264 if (RetVT.SimpleTy != MVT::v4i16)
6265 return 0;
6266 if ((Subtarget->hasNEON())) {
6267 return fastEmitInst_rr(AArch64::TRN1v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6268 }
6269 return 0;
6270}
6271
6272unsigned fastEmit_AArch64ISD_TRN1_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6273 if (RetVT.SimpleTy != MVT::v8i16)
6274 return 0;
6275 if ((Subtarget->hasNEON())) {
6276 return fastEmitInst_rr(AArch64::TRN1v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6277 }
6278 return 0;
6279}
6280
6281unsigned fastEmit_AArch64ISD_TRN1_MVT_v2i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6282 if (RetVT.SimpleTy != MVT::v2i32)
6283 return 0;
6284 if ((Subtarget->hasNEON())) {
6285 return fastEmitInst_rr(AArch64::TRN1v2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6286 }
6287 return 0;
6288}
6289
6290unsigned fastEmit_AArch64ISD_TRN1_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6291 if (RetVT.SimpleTy != MVT::v4i32)
6292 return 0;
6293 if ((Subtarget->hasNEON())) {
6294 return fastEmitInst_rr(AArch64::TRN1v4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6295 }
6296 return 0;
6297}
6298
6299unsigned fastEmit_AArch64ISD_TRN1_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6300 if (RetVT.SimpleTy != MVT::v2i64)
6301 return 0;
6302 if ((Subtarget->hasNEON())) {
6303 return fastEmitInst_rr(AArch64::TRN1v2i64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6304 }
6305 return 0;
6306}
6307
6308unsigned fastEmit_AArch64ISD_TRN1_MVT_v4f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6309 if (RetVT.SimpleTy != MVT::v4f16)
6310 return 0;
6311 if ((Subtarget->hasNEON())) {
6312 return fastEmitInst_rr(AArch64::TRN1v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6313 }
6314 return 0;
6315}
6316
6317unsigned fastEmit_AArch64ISD_TRN1_MVT_v8f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6318 if (RetVT.SimpleTy != MVT::v8f16)
6319 return 0;
6320 if ((Subtarget->hasNEON())) {
6321 return fastEmitInst_rr(AArch64::TRN1v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6322 }
6323 return 0;
6324}
6325
6326unsigned fastEmit_AArch64ISD_TRN1_MVT_v2f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6327 if (RetVT.SimpleTy != MVT::v2f32)
6328 return 0;
6329 if ((Subtarget->hasNEON())) {
6330 return fastEmitInst_rr(AArch64::TRN1v2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6331 }
6332 return 0;
6333}
6334
6335unsigned fastEmit_AArch64ISD_TRN1_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6336 if (RetVT.SimpleTy != MVT::v4f32)
6337 return 0;
6338 if ((Subtarget->hasNEON())) {
6339 return fastEmitInst_rr(AArch64::TRN1v4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6340 }
6341 return 0;
6342}
6343
6344unsigned fastEmit_AArch64ISD_TRN1_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6345 if (RetVT.SimpleTy != MVT::v2f64)
6346 return 0;
6347 if ((Subtarget->hasNEON())) {
6348 return fastEmitInst_rr(AArch64::TRN1v2i64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6349 }
6350 return 0;
6351}
6352
6353unsigned fastEmit_AArch64ISD_TRN1_MVT_nxv2i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6354 if (RetVT.SimpleTy != MVT::nxv2i1)
6355 return 0;
6356 if ((Subtarget->hasSVE())) {
6357 return fastEmitInst_rr(AArch64::TRN1_PPP_D, &AArch64::PPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6358 }
6359 return 0;
6360}
6361
6362unsigned fastEmit_AArch64ISD_TRN1_MVT_nxv4i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6363 if (RetVT.SimpleTy != MVT::nxv4i1)
6364 return 0;
6365 if ((Subtarget->hasSVE())) {
6366 return fastEmitInst_rr(AArch64::TRN1_PPP_S, &AArch64::PPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6367 }
6368 return 0;
6369}
6370
6371unsigned fastEmit_AArch64ISD_TRN1_MVT_nxv8i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6372 if (RetVT.SimpleTy != MVT::nxv8i1)
6373 return 0;
6374 if ((Subtarget->hasSVE())) {
6375 return fastEmitInst_rr(AArch64::TRN1_PPP_H, &AArch64::PPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6376 }
6377 return 0;
6378}
6379
6380unsigned fastEmit_AArch64ISD_TRN1_MVT_nxv16i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6381 if (RetVT.SimpleTy != MVT::nxv16i1)
6382 return 0;
6383 if ((Subtarget->hasSVE())) {
6384 return fastEmitInst_rr(AArch64::TRN1_PPP_B, &AArch64::PPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6385 }
6386 return 0;
6387}
6388
6389unsigned fastEmit_AArch64ISD_TRN1_MVT_nxv16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6390 if (RetVT.SimpleTy != MVT::nxv16i8)
6391 return 0;
6392 if ((Subtarget->hasSVE())) {
6393 return fastEmitInst_rr(AArch64::TRN1_ZZZ_B, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6394 }
6395 return 0;
6396}
6397
6398unsigned fastEmit_AArch64ISD_TRN1_MVT_nxv8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6399 if (RetVT.SimpleTy != MVT::nxv8i16)
6400 return 0;
6401 if ((Subtarget->hasSVE())) {
6402 return fastEmitInst_rr(AArch64::TRN1_ZZZ_H, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6403 }
6404 return 0;
6405}
6406
6407unsigned fastEmit_AArch64ISD_TRN1_MVT_nxv4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6408 if (RetVT.SimpleTy != MVT::nxv4i32)
6409 return 0;
6410 if ((Subtarget->hasSVE())) {
6411 return fastEmitInst_rr(AArch64::TRN1_ZZZ_S, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6412 }
6413 return 0;
6414}
6415
6416unsigned fastEmit_AArch64ISD_TRN1_MVT_nxv2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6417 if (RetVT.SimpleTy != MVT::nxv2i64)
6418 return 0;
6419 if ((Subtarget->hasSVE())) {
6420 return fastEmitInst_rr(AArch64::TRN1_ZZZ_D, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6421 }
6422 return 0;
6423}
6424
6425unsigned fastEmit_AArch64ISD_TRN1_MVT_nxv2f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6426 if (RetVT.SimpleTy != MVT::nxv2f16)
6427 return 0;
6428 if ((Subtarget->hasSVE())) {
6429 return fastEmitInst_rr(AArch64::TRN1_ZZZ_D, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6430 }
6431 return 0;
6432}
6433
6434unsigned fastEmit_AArch64ISD_TRN1_MVT_nxv4f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6435 if (RetVT.SimpleTy != MVT::nxv4f16)
6436 return 0;
6437 if ((Subtarget->hasSVE())) {
6438 return fastEmitInst_rr(AArch64::TRN1_ZZZ_S, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6439 }
6440 return 0;
6441}
6442
6443unsigned fastEmit_AArch64ISD_TRN1_MVT_nxv8f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6444 if (RetVT.SimpleTy != MVT::nxv8f16)
6445 return 0;
6446 if ((Subtarget->hasSVE())) {
6447 return fastEmitInst_rr(AArch64::TRN1_ZZZ_H, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6448 }
6449 return 0;
6450}
6451
6452unsigned fastEmit_AArch64ISD_TRN1_MVT_nxv8bf16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6453 if (RetVT.SimpleTy != MVT::nxv8bf16)
6454 return 0;
6455 if ((Subtarget->hasSVE())) {
6456 return fastEmitInst_rr(AArch64::TRN1_ZZZ_H, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6457 }
6458 return 0;
6459}
6460
6461unsigned fastEmit_AArch64ISD_TRN1_MVT_nxv2f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6462 if (RetVT.SimpleTy != MVT::nxv2f32)
6463 return 0;
6464 if ((Subtarget->hasSVE())) {
6465 return fastEmitInst_rr(AArch64::TRN1_ZZZ_D, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6466 }
6467 return 0;
6468}
6469
6470unsigned fastEmit_AArch64ISD_TRN1_MVT_nxv4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6471 if (RetVT.SimpleTy != MVT::nxv4f32)
6472 return 0;
6473 if ((Subtarget->hasSVE())) {
6474 return fastEmitInst_rr(AArch64::TRN1_ZZZ_S, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6475 }
6476 return 0;
6477}
6478
6479unsigned fastEmit_AArch64ISD_TRN1_MVT_nxv2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6480 if (RetVT.SimpleTy != MVT::nxv2f64)
6481 return 0;
6482 if ((Subtarget->hasSVE())) {
6483 return fastEmitInst_rr(AArch64::TRN1_ZZZ_D, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6484 }
6485 return 0;
6486}
6487
6488unsigned fastEmit_AArch64ISD_TRN1_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6489 switch (VT.SimpleTy) {
6490 case MVT::v8i8: return fastEmit_AArch64ISD_TRN1_MVT_v8i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6491 case MVT::v16i8: return fastEmit_AArch64ISD_TRN1_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6492 case MVT::v4i16: return fastEmit_AArch64ISD_TRN1_MVT_v4i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6493 case MVT::v8i16: return fastEmit_AArch64ISD_TRN1_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6494 case MVT::v2i32: return fastEmit_AArch64ISD_TRN1_MVT_v2i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6495 case MVT::v4i32: return fastEmit_AArch64ISD_TRN1_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6496 case MVT::v2i64: return fastEmit_AArch64ISD_TRN1_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6497 case MVT::v4f16: return fastEmit_AArch64ISD_TRN1_MVT_v4f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6498 case MVT::v8f16: return fastEmit_AArch64ISD_TRN1_MVT_v8f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6499 case MVT::v2f32: return fastEmit_AArch64ISD_TRN1_MVT_v2f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6500 case MVT::v4f32: return fastEmit_AArch64ISD_TRN1_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6501 case MVT::v2f64: return fastEmit_AArch64ISD_TRN1_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6502 case MVT::nxv2i1: return fastEmit_AArch64ISD_TRN1_MVT_nxv2i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6503 case MVT::nxv4i1: return fastEmit_AArch64ISD_TRN1_MVT_nxv4i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6504 case MVT::nxv8i1: return fastEmit_AArch64ISD_TRN1_MVT_nxv8i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6505 case MVT::nxv16i1: return fastEmit_AArch64ISD_TRN1_MVT_nxv16i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6506 case MVT::nxv16i8: return fastEmit_AArch64ISD_TRN1_MVT_nxv16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6507 case MVT::nxv8i16: return fastEmit_AArch64ISD_TRN1_MVT_nxv8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6508 case MVT::nxv4i32: return fastEmit_AArch64ISD_TRN1_MVT_nxv4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6509 case MVT::nxv2i64: return fastEmit_AArch64ISD_TRN1_MVT_nxv2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6510 case MVT::nxv2f16: return fastEmit_AArch64ISD_TRN1_MVT_nxv2f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6511 case MVT::nxv4f16: return fastEmit_AArch64ISD_TRN1_MVT_nxv4f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6512 case MVT::nxv8f16: return fastEmit_AArch64ISD_TRN1_MVT_nxv8f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6513 case MVT::nxv8bf16: return fastEmit_AArch64ISD_TRN1_MVT_nxv8bf16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6514 case MVT::nxv2f32: return fastEmit_AArch64ISD_TRN1_MVT_nxv2f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6515 case MVT::nxv4f32: return fastEmit_AArch64ISD_TRN1_MVT_nxv4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6516 case MVT::nxv2f64: return fastEmit_AArch64ISD_TRN1_MVT_nxv2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6517 default: return 0;
6518 }
6519}
6520
6521// FastEmit functions for AArch64ISD::TRN2.
6522
6523unsigned fastEmit_AArch64ISD_TRN2_MVT_v8i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6524 if (RetVT.SimpleTy != MVT::v8i8)
6525 return 0;
6526 if ((Subtarget->hasNEON())) {
6527 return fastEmitInst_rr(AArch64::TRN2v8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6528 }
6529 return 0;
6530}
6531
6532unsigned fastEmit_AArch64ISD_TRN2_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6533 if (RetVT.SimpleTy != MVT::v16i8)
6534 return 0;
6535 if ((Subtarget->hasNEON())) {
6536 return fastEmitInst_rr(AArch64::TRN2v16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6537 }
6538 return 0;
6539}
6540
6541unsigned fastEmit_AArch64ISD_TRN2_MVT_v4i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6542 if (RetVT.SimpleTy != MVT::v4i16)
6543 return 0;
6544 if ((Subtarget->hasNEON())) {
6545 return fastEmitInst_rr(AArch64::TRN2v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6546 }
6547 return 0;
6548}
6549
6550unsigned fastEmit_AArch64ISD_TRN2_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6551 if (RetVT.SimpleTy != MVT::v8i16)
6552 return 0;
6553 if ((Subtarget->hasNEON())) {
6554 return fastEmitInst_rr(AArch64::TRN2v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6555 }
6556 return 0;
6557}
6558
6559unsigned fastEmit_AArch64ISD_TRN2_MVT_v2i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6560 if (RetVT.SimpleTy != MVT::v2i32)
6561 return 0;
6562 if ((Subtarget->hasNEON())) {
6563 return fastEmitInst_rr(AArch64::TRN2v2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6564 }
6565 return 0;
6566}
6567
6568unsigned fastEmit_AArch64ISD_TRN2_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6569 if (RetVT.SimpleTy != MVT::v4i32)
6570 return 0;
6571 if ((Subtarget->hasNEON())) {
6572 return fastEmitInst_rr(AArch64::TRN2v4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6573 }
6574 return 0;
6575}
6576
6577unsigned fastEmit_AArch64ISD_TRN2_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6578 if (RetVT.SimpleTy != MVT::v2i64)
6579 return 0;
6580 if ((Subtarget->hasNEON())) {
6581 return fastEmitInst_rr(AArch64::TRN2v2i64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6582 }
6583 return 0;
6584}
6585
6586unsigned fastEmit_AArch64ISD_TRN2_MVT_v4f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6587 if (RetVT.SimpleTy != MVT::v4f16)
6588 return 0;
6589 if ((Subtarget->hasNEON())) {
6590 return fastEmitInst_rr(AArch64::TRN2v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6591 }
6592 return 0;
6593}
6594
6595unsigned fastEmit_AArch64ISD_TRN2_MVT_v8f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6596 if (RetVT.SimpleTy != MVT::v8f16)
6597 return 0;
6598 if ((Subtarget->hasNEON())) {
6599 return fastEmitInst_rr(AArch64::TRN2v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6600 }
6601 return 0;
6602}
6603
6604unsigned fastEmit_AArch64ISD_TRN2_MVT_v2f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6605 if (RetVT.SimpleTy != MVT::v2f32)
6606 return 0;
6607 if ((Subtarget->hasNEON())) {
6608 return fastEmitInst_rr(AArch64::TRN2v2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6609 }
6610 return 0;
6611}
6612
6613unsigned fastEmit_AArch64ISD_TRN2_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6614 if (RetVT.SimpleTy != MVT::v4f32)
6615 return 0;
6616 if ((Subtarget->hasNEON())) {
6617 return fastEmitInst_rr(AArch64::TRN2v4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6618 }
6619 return 0;
6620}
6621
6622unsigned fastEmit_AArch64ISD_TRN2_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6623 if (RetVT.SimpleTy != MVT::v2f64)
6624 return 0;
6625 if ((Subtarget->hasNEON())) {
6626 return fastEmitInst_rr(AArch64::TRN2v2i64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6627 }
6628 return 0;
6629}
6630
6631unsigned fastEmit_AArch64ISD_TRN2_MVT_nxv2i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6632 if (RetVT.SimpleTy != MVT::nxv2i1)
6633 return 0;
6634 if ((Subtarget->hasSVE())) {
6635 return fastEmitInst_rr(AArch64::TRN2_PPP_D, &AArch64::PPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6636 }
6637 return 0;
6638}
6639
6640unsigned fastEmit_AArch64ISD_TRN2_MVT_nxv4i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6641 if (RetVT.SimpleTy != MVT::nxv4i1)
6642 return 0;
6643 if ((Subtarget->hasSVE())) {
6644 return fastEmitInst_rr(AArch64::TRN2_PPP_S, &AArch64::PPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6645 }
6646 return 0;
6647}
6648
6649unsigned fastEmit_AArch64ISD_TRN2_MVT_nxv8i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6650 if (RetVT.SimpleTy != MVT::nxv8i1)
6651 return 0;
6652 if ((Subtarget->hasSVE())) {
6653 return fastEmitInst_rr(AArch64::TRN2_PPP_H, &AArch64::PPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6654 }
6655 return 0;
6656}
6657
6658unsigned fastEmit_AArch64ISD_TRN2_MVT_nxv16i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6659 if (RetVT.SimpleTy != MVT::nxv16i1)
6660 return 0;
6661 if ((Subtarget->hasSVE())) {
6662 return fastEmitInst_rr(AArch64::TRN2_PPP_B, &AArch64::PPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6663 }
6664 return 0;
6665}
6666
6667unsigned fastEmit_AArch64ISD_TRN2_MVT_nxv16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6668 if (RetVT.SimpleTy != MVT::nxv16i8)
6669 return 0;
6670 if ((Subtarget->hasSVE())) {
6671 return fastEmitInst_rr(AArch64::TRN2_ZZZ_B, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6672 }
6673 return 0;
6674}
6675
6676unsigned fastEmit_AArch64ISD_TRN2_MVT_nxv8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6677 if (RetVT.SimpleTy != MVT::nxv8i16)
6678 return 0;
6679 if ((Subtarget->hasSVE())) {
6680 return fastEmitInst_rr(AArch64::TRN2_ZZZ_H, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6681 }
6682 return 0;
6683}
6684
6685unsigned fastEmit_AArch64ISD_TRN2_MVT_nxv4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6686 if (RetVT.SimpleTy != MVT::nxv4i32)
6687 return 0;
6688 if ((Subtarget->hasSVE())) {
6689 return fastEmitInst_rr(AArch64::TRN2_ZZZ_S, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6690 }
6691 return 0;
6692}
6693
6694unsigned fastEmit_AArch64ISD_TRN2_MVT_nxv2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6695 if (RetVT.SimpleTy != MVT::nxv2i64)
6696 return 0;
6697 if ((Subtarget->hasSVE())) {
6698 return fastEmitInst_rr(AArch64::TRN2_ZZZ_D, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6699 }
6700 return 0;
6701}
6702
6703unsigned fastEmit_AArch64ISD_TRN2_MVT_nxv2f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6704 if (RetVT.SimpleTy != MVT::nxv2f16)
6705 return 0;
6706 if ((Subtarget->hasSVE())) {
6707 return fastEmitInst_rr(AArch64::TRN2_ZZZ_D, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6708 }
6709 return 0;
6710}
6711
6712unsigned fastEmit_AArch64ISD_TRN2_MVT_nxv4f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6713 if (RetVT.SimpleTy != MVT::nxv4f16)
6714 return 0;
6715 if ((Subtarget->hasSVE())) {
6716 return fastEmitInst_rr(AArch64::TRN2_ZZZ_S, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6717 }
6718 return 0;
6719}
6720
6721unsigned fastEmit_AArch64ISD_TRN2_MVT_nxv8f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6722 if (RetVT.SimpleTy != MVT::nxv8f16)
6723 return 0;
6724 if ((Subtarget->hasSVE())) {
6725 return fastEmitInst_rr(AArch64::TRN2_ZZZ_H, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6726 }
6727 return 0;
6728}
6729
6730unsigned fastEmit_AArch64ISD_TRN2_MVT_nxv8bf16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6731 if (RetVT.SimpleTy != MVT::nxv8bf16)
6732 return 0;
6733 if ((Subtarget->hasSVE())) {
6734 return fastEmitInst_rr(AArch64::TRN2_ZZZ_H, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6735 }
6736 return 0;
6737}
6738
6739unsigned fastEmit_AArch64ISD_TRN2_MVT_nxv2f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6740 if (RetVT.SimpleTy != MVT::nxv2f32)
6741 return 0;
6742 if ((Subtarget->hasSVE())) {
6743 return fastEmitInst_rr(AArch64::TRN2_ZZZ_D, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6744 }
6745 return 0;
6746}
6747
6748unsigned fastEmit_AArch64ISD_TRN2_MVT_nxv4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6749 if (RetVT.SimpleTy != MVT::nxv4f32)
6750 return 0;
6751 if ((Subtarget->hasSVE())) {
6752 return fastEmitInst_rr(AArch64::TRN2_ZZZ_S, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6753 }
6754 return 0;
6755}
6756
6757unsigned fastEmit_AArch64ISD_TRN2_MVT_nxv2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6758 if (RetVT.SimpleTy != MVT::nxv2f64)
6759 return 0;
6760 if ((Subtarget->hasSVE())) {
6761 return fastEmitInst_rr(AArch64::TRN2_ZZZ_D, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6762 }
6763 return 0;
6764}
6765
6766unsigned fastEmit_AArch64ISD_TRN2_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6767 switch (VT.SimpleTy) {
6768 case MVT::v8i8: return fastEmit_AArch64ISD_TRN2_MVT_v8i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6769 case MVT::v16i8: return fastEmit_AArch64ISD_TRN2_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6770 case MVT::v4i16: return fastEmit_AArch64ISD_TRN2_MVT_v4i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6771 case MVT::v8i16: return fastEmit_AArch64ISD_TRN2_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6772 case MVT::v2i32: return fastEmit_AArch64ISD_TRN2_MVT_v2i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6773 case MVT::v4i32: return fastEmit_AArch64ISD_TRN2_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6774 case MVT::v2i64: return fastEmit_AArch64ISD_TRN2_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6775 case MVT::v4f16: return fastEmit_AArch64ISD_TRN2_MVT_v4f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6776 case MVT::v8f16: return fastEmit_AArch64ISD_TRN2_MVT_v8f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6777 case MVT::v2f32: return fastEmit_AArch64ISD_TRN2_MVT_v2f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6778 case MVT::v4f32: return fastEmit_AArch64ISD_TRN2_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6779 case MVT::v2f64: return fastEmit_AArch64ISD_TRN2_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6780 case MVT::nxv2i1: return fastEmit_AArch64ISD_TRN2_MVT_nxv2i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6781 case MVT::nxv4i1: return fastEmit_AArch64ISD_TRN2_MVT_nxv4i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6782 case MVT::nxv8i1: return fastEmit_AArch64ISD_TRN2_MVT_nxv8i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6783 case MVT::nxv16i1: return fastEmit_AArch64ISD_TRN2_MVT_nxv16i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6784 case MVT::nxv16i8: return fastEmit_AArch64ISD_TRN2_MVT_nxv16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6785 case MVT::nxv8i16: return fastEmit_AArch64ISD_TRN2_MVT_nxv8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6786 case MVT::nxv4i32: return fastEmit_AArch64ISD_TRN2_MVT_nxv4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6787 case MVT::nxv2i64: return fastEmit_AArch64ISD_TRN2_MVT_nxv2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6788 case MVT::nxv2f16: return fastEmit_AArch64ISD_TRN2_MVT_nxv2f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6789 case MVT::nxv4f16: return fastEmit_AArch64ISD_TRN2_MVT_nxv4f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6790 case MVT::nxv8f16: return fastEmit_AArch64ISD_TRN2_MVT_nxv8f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6791 case MVT::nxv8bf16: return fastEmit_AArch64ISD_TRN2_MVT_nxv8bf16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6792 case MVT::nxv2f32: return fastEmit_AArch64ISD_TRN2_MVT_nxv2f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6793 case MVT::nxv4f32: return fastEmit_AArch64ISD_TRN2_MVT_nxv4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6794 case MVT::nxv2f64: return fastEmit_AArch64ISD_TRN2_MVT_nxv2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6795 default: return 0;
6796 }
6797}
6798
6799// FastEmit functions for AArch64ISD::UABD.
6800
6801unsigned fastEmit_AArch64ISD_UABD_MVT_v8i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6802 if (RetVT.SimpleTy != MVT::v8i8)
6803 return 0;
6804 if ((Subtarget->hasNEON())) {
6805 return fastEmitInst_rr(AArch64::UABDv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6806 }
6807 return 0;
6808}
6809
6810unsigned fastEmit_AArch64ISD_UABD_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6811 if (RetVT.SimpleTy != MVT::v16i8)
6812 return 0;
6813 if ((Subtarget->hasNEON())) {
6814 return fastEmitInst_rr(AArch64::UABDv16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6815 }
6816 return 0;
6817}
6818
6819unsigned fastEmit_AArch64ISD_UABD_MVT_v4i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6820 if (RetVT.SimpleTy != MVT::v4i16)
6821 return 0;
6822 if ((Subtarget->hasNEON())) {
6823 return fastEmitInst_rr(AArch64::UABDv4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6824 }
6825 return 0;
6826}
6827
6828unsigned fastEmit_AArch64ISD_UABD_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6829 if (RetVT.SimpleTy != MVT::v8i16)
6830 return 0;
6831 if ((Subtarget->hasNEON())) {
6832 return fastEmitInst_rr(AArch64::UABDv8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6833 }
6834 return 0;
6835}
6836
6837unsigned fastEmit_AArch64ISD_UABD_MVT_v2i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6838 if (RetVT.SimpleTy != MVT::v2i32)
6839 return 0;
6840 if ((Subtarget->hasNEON())) {
6841 return fastEmitInst_rr(AArch64::UABDv2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6842 }
6843 return 0;
6844}
6845
6846unsigned fastEmit_AArch64ISD_UABD_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6847 if (RetVT.SimpleTy != MVT::v4i32)
6848 return 0;
6849 if ((Subtarget->hasNEON())) {
6850 return fastEmitInst_rr(AArch64::UABDv4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6851 }
6852 return 0;
6853}
6854
6855unsigned fastEmit_AArch64ISD_UABD_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6856 switch (VT.SimpleTy) {
6857 case MVT::v8i8: return fastEmit_AArch64ISD_UABD_MVT_v8i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6858 case MVT::v16i8: return fastEmit_AArch64ISD_UABD_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6859 case MVT::v4i16: return fastEmit_AArch64ISD_UABD_MVT_v4i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6860 case MVT::v8i16: return fastEmit_AArch64ISD_UABD_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6861 case MVT::v2i32: return fastEmit_AArch64ISD_UABD_MVT_v2i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6862 case MVT::v4i32: return fastEmit_AArch64ISD_UABD_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6863 default: return 0;
6864 }
6865}
6866
6867// FastEmit functions for AArch64ISD::UHADD.
6868
6869unsigned fastEmit_AArch64ISD_UHADD_MVT_v8i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6870 if (RetVT.SimpleTy != MVT::v8i8)
6871 return 0;
6872 if ((Subtarget->hasNEON())) {
6873 return fastEmitInst_rr(AArch64::UHADDv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6874 }
6875 return 0;
6876}
6877
6878unsigned fastEmit_AArch64ISD_UHADD_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6879 if (RetVT.SimpleTy != MVT::v16i8)
6880 return 0;
6881 if ((Subtarget->hasNEON())) {
6882 return fastEmitInst_rr(AArch64::UHADDv16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6883 }
6884 return 0;
6885}
6886
6887unsigned fastEmit_AArch64ISD_UHADD_MVT_v4i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6888 if (RetVT.SimpleTy != MVT::v4i16)
6889 return 0;
6890 if ((Subtarget->hasNEON())) {
6891 return fastEmitInst_rr(AArch64::UHADDv4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6892 }
6893 return 0;
6894}
6895
6896unsigned fastEmit_AArch64ISD_UHADD_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6897 if (RetVT.SimpleTy != MVT::v8i16)
6898 return 0;
6899 if ((Subtarget->hasNEON())) {
6900 return fastEmitInst_rr(AArch64::UHADDv8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6901 }
6902 return 0;
6903}
6904
6905unsigned fastEmit_AArch64ISD_UHADD_MVT_v2i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6906 if (RetVT.SimpleTy != MVT::v2i32)
6907 return 0;
6908 if ((Subtarget->hasNEON())) {
6909 return fastEmitInst_rr(AArch64::UHADDv2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6910 }
6911 return 0;
6912}
6913
6914unsigned fastEmit_AArch64ISD_UHADD_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6915 if (RetVT.SimpleTy != MVT::v4i32)
6916 return 0;
6917 if ((Subtarget->hasNEON())) {
6918 return fastEmitInst_rr(AArch64::UHADDv4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6919 }
6920 return 0;
6921}
6922
6923unsigned fastEmit_AArch64ISD_UHADD_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6924 switch (VT.SimpleTy) {
6925 case MVT::v8i8: return fastEmit_AArch64ISD_UHADD_MVT_v8i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6926 case MVT::v16i8: return fastEmit_AArch64ISD_UHADD_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6927 case MVT::v4i16: return fastEmit_AArch64ISD_UHADD_MVT_v4i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6928 case MVT::v8i16: return fastEmit_AArch64ISD_UHADD_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6929 case MVT::v2i32: return fastEmit_AArch64ISD_UHADD_MVT_v2i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6930 case MVT::v4i32: return fastEmit_AArch64ISD_UHADD_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6931 default: return 0;
6932 }
6933}
6934
6935// FastEmit functions for AArch64ISD::UMULL.
6936
6937unsigned fastEmit_AArch64ISD_UMULL_MVT_v8i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6938 if (RetVT.SimpleTy != MVT::v8i16)
6939 return 0;
6940 return fastEmitInst_rr(AArch64::UMULLv8i8_v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6941}
6942
6943unsigned fastEmit_AArch64ISD_UMULL_MVT_v4i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6944 if (RetVT.SimpleTy != MVT::v4i32)
6945 return 0;
6946 return fastEmitInst_rr(AArch64::UMULLv4i16_v4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6947}
6948
6949unsigned fastEmit_AArch64ISD_UMULL_MVT_v2i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6950 if (RetVT.SimpleTy != MVT::v2i64)
6951 return 0;
6952 return fastEmitInst_rr(AArch64::UMULLv2i32_v2i64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6953}
6954
6955unsigned fastEmit_AArch64ISD_UMULL_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6956 switch (VT.SimpleTy) {
6957 case MVT::v8i8: return fastEmit_AArch64ISD_UMULL_MVT_v8i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6958 case MVT::v4i16: return fastEmit_AArch64ISD_UMULL_MVT_v4i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6959 case MVT::v2i32: return fastEmit_AArch64ISD_UMULL_MVT_v2i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6960 default: return 0;
6961 }
6962}
6963
6964// FastEmit functions for AArch64ISD::URHADD.
6965
6966unsigned fastEmit_AArch64ISD_URHADD_MVT_v8i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6967 if (RetVT.SimpleTy != MVT::v8i8)
6968 return 0;
6969 if ((Subtarget->hasNEON())) {
6970 return fastEmitInst_rr(AArch64::URHADDv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6971 }
6972 return 0;
6973}
6974
6975unsigned fastEmit_AArch64ISD_URHADD_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6976 if (RetVT.SimpleTy != MVT::v16i8)
6977 return 0;
6978 if ((Subtarget->hasNEON())) {
6979 return fastEmitInst_rr(AArch64::URHADDv16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6980 }
6981 return 0;
6982}
6983
6984unsigned fastEmit_AArch64ISD_URHADD_MVT_v4i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6985 if (RetVT.SimpleTy != MVT::v4i16)
6986 return 0;
6987 if ((Subtarget->hasNEON())) {
6988 return fastEmitInst_rr(AArch64::URHADDv4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6989 }
6990 return 0;
6991}
6992
6993unsigned fastEmit_AArch64ISD_URHADD_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6994 if (RetVT.SimpleTy != MVT::v8i16)
6995 return 0;
6996 if ((Subtarget->hasNEON())) {
6997 return fastEmitInst_rr(AArch64::URHADDv8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6998 }
6999 return 0;
7000}
7001
7002unsigned fastEmit_AArch64ISD_URHADD_MVT_v2i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7003 if (RetVT.SimpleTy != MVT::v2i32)
7004 return 0;
7005 if ((Subtarget->hasNEON())) {
7006 return fastEmitInst_rr(AArch64::URHADDv2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7007 }
7008 return 0;
7009}
7010
7011unsigned fastEmit_AArch64ISD_URHADD_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7012 if (RetVT.SimpleTy != MVT::v4i32)
7013 return 0;
7014 if ((Subtarget->hasNEON())) {
7015 return fastEmitInst_rr(AArch64::URHADDv4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7016 }
7017 return 0;
7018}
7019
7020unsigned fastEmit_AArch64ISD_URHADD_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7021 switch (VT.SimpleTy) {
7022 case MVT::v8i8: return fastEmit_AArch64ISD_URHADD_MVT_v8i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7023 case MVT::v16i8: return fastEmit_AArch64ISD_URHADD_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7024 case MVT::v4i16: return fastEmit_AArch64ISD_URHADD_MVT_v4i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7025 case MVT::v8i16: return fastEmit_AArch64ISD_URHADD_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7026 case MVT::v2i32: return fastEmit_AArch64ISD_URHADD_MVT_v2i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7027 case MVT::v4i32: return fastEmit_AArch64ISD_URHADD_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7028 default: return 0;
7029 }
7030}
7031
7032// FastEmit functions for AArch64ISD::UZP1.
7033
7034unsigned fastEmit_AArch64ISD_UZP1_MVT_v8i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7035 if (RetVT.SimpleTy != MVT::v8i8)
7036 return 0;
7037 if ((Subtarget->hasNEON())) {
7038 return fastEmitInst_rr(AArch64::UZP1v8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7039 }
7040 return 0;
7041}
7042
7043unsigned fastEmit_AArch64ISD_UZP1_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7044 if (RetVT.SimpleTy != MVT::v16i8)
7045 return 0;
7046 if ((Subtarget->hasNEON())) {
7047 return fastEmitInst_rr(AArch64::UZP1v16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7048 }
7049 return 0;
7050}
7051
7052unsigned fastEmit_AArch64ISD_UZP1_MVT_v4i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7053 if (RetVT.SimpleTy != MVT::v4i16)
7054 return 0;
7055 if ((Subtarget->hasNEON())) {
7056 return fastEmitInst_rr(AArch64::UZP1v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7057 }
7058 return 0;
7059}
7060
7061unsigned fastEmit_AArch64ISD_UZP1_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7062 if (RetVT.SimpleTy != MVT::v8i16)
7063 return 0;
7064 if ((Subtarget->hasNEON())) {
7065 return fastEmitInst_rr(AArch64::UZP1v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7066 }
7067 return 0;
7068}
7069
7070unsigned fastEmit_AArch64ISD_UZP1_MVT_v2i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7071 if (RetVT.SimpleTy != MVT::v2i32)
7072 return 0;
7073 if ((Subtarget->hasNEON())) {
7074 return fastEmitInst_rr(AArch64::UZP1v2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7075 }
7076 return 0;
7077}
7078
7079unsigned fastEmit_AArch64ISD_UZP1_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7080 if (RetVT.SimpleTy != MVT::v4i32)
7081 return 0;
7082 if ((Subtarget->hasNEON())) {
7083 return fastEmitInst_rr(AArch64::UZP1v4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7084 }
7085 return 0;
7086}
7087
7088unsigned fastEmit_AArch64ISD_UZP1_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7089 if (RetVT.SimpleTy != MVT::v2i64)
7090 return 0;
7091 if ((Subtarget->hasNEON())) {
7092 return fastEmitInst_rr(AArch64::UZP1v2i64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7093 }
7094 return 0;
7095}
7096
7097unsigned fastEmit_AArch64ISD_UZP1_MVT_v4f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7098 if (RetVT.SimpleTy != MVT::v4f16)
7099 return 0;
7100 if ((Subtarget->hasNEON())) {
7101 return fastEmitInst_rr(AArch64::UZP1v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7102 }
7103 return 0;
7104}
7105
7106unsigned fastEmit_AArch64ISD_UZP1_MVT_v8f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7107 if (RetVT.SimpleTy != MVT::v8f16)
7108 return 0;
7109 if ((Subtarget->hasNEON())) {
7110 return fastEmitInst_rr(AArch64::UZP1v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7111 }
7112 return 0;
7113}
7114
7115unsigned fastEmit_AArch64ISD_UZP1_MVT_v2f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7116 if (RetVT.SimpleTy != MVT::v2f32)
7117 return 0;
7118 if ((Subtarget->hasNEON())) {
7119 return fastEmitInst_rr(AArch64::UZP1v2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7120 }
7121 return 0;
7122}
7123
7124unsigned fastEmit_AArch64ISD_UZP1_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7125 if (RetVT.SimpleTy != MVT::v4f32)
7126 return 0;
7127 if ((Subtarget->hasNEON())) {
7128 return fastEmitInst_rr(AArch64::UZP1v4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7129 }
7130 return 0;
7131}
7132
7133unsigned fastEmit_AArch64ISD_UZP1_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7134 if (RetVT.SimpleTy != MVT::v2f64)
7135 return 0;
7136 if ((Subtarget->hasNEON())) {
7137 return fastEmitInst_rr(AArch64::UZP1v2i64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7138 }
7139 return 0;
7140}
7141
7142unsigned fastEmit_AArch64ISD_UZP1_MVT_nxv2i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7143 if (RetVT.SimpleTy != MVT::nxv2i1)
7144 return 0;
7145 if ((Subtarget->hasSVE())) {
7146 return fastEmitInst_rr(AArch64::UZP1_PPP_D, &AArch64::PPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7147 }
7148 return 0;
7149}
7150
7151unsigned fastEmit_AArch64ISD_UZP1_MVT_nxv4i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7152 if (RetVT.SimpleTy != MVT::nxv4i1)
7153 return 0;
7154 if ((Subtarget->hasSVE())) {
7155 return fastEmitInst_rr(AArch64::UZP1_PPP_S, &AArch64::PPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7156 }
7157 return 0;
7158}
7159
7160unsigned fastEmit_AArch64ISD_UZP1_MVT_nxv8i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7161 if (RetVT.SimpleTy != MVT::nxv8i1)
7162 return 0;
7163 if ((Subtarget->hasSVE())) {
7164 return fastEmitInst_rr(AArch64::UZP1_PPP_H, &AArch64::PPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7165 }
7166 return 0;
7167}
7168
7169unsigned fastEmit_AArch64ISD_UZP1_MVT_nxv16i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7170 if (RetVT.SimpleTy != MVT::nxv16i1)
7171 return 0;
7172 if ((Subtarget->hasSVE())) {
7173 return fastEmitInst_rr(AArch64::UZP1_PPP_B, &AArch64::PPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7174 }
7175 return 0;
7176}
7177
7178unsigned fastEmit_AArch64ISD_UZP1_MVT_nxv16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7179 if (RetVT.SimpleTy != MVT::nxv16i8)
7180 return 0;
7181 if ((Subtarget->hasSVE())) {
7182 return fastEmitInst_rr(AArch64::UZP1_ZZZ_B, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7183 }
7184 return 0;
7185}
7186
7187unsigned fastEmit_AArch64ISD_UZP1_MVT_nxv8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7188 if (RetVT.SimpleTy != MVT::nxv8i16)
7189 return 0;
7190 if ((Subtarget->hasSVE())) {
7191 return fastEmitInst_rr(AArch64::UZP1_ZZZ_H, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7192 }
7193 return 0;
7194}
7195
7196unsigned fastEmit_AArch64ISD_UZP1_MVT_nxv4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7197 if (RetVT.SimpleTy != MVT::nxv4i32)
7198 return 0;
7199 if ((Subtarget->hasSVE())) {
7200 return fastEmitInst_rr(AArch64::UZP1_ZZZ_S, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7201 }
7202 return 0;
7203}
7204
7205unsigned fastEmit_AArch64ISD_UZP1_MVT_nxv2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7206 if (RetVT.SimpleTy != MVT::nxv2i64)
7207 return 0;
7208 if ((Subtarget->hasSVE())) {
7209 return fastEmitInst_rr(AArch64::UZP1_ZZZ_D, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7210 }
7211 return 0;
7212}
7213
7214unsigned fastEmit_AArch64ISD_UZP1_MVT_nxv2f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7215 if (RetVT.SimpleTy != MVT::nxv2f16)
7216 return 0;
7217 if ((Subtarget->hasSVE())) {
7218 return fastEmitInst_rr(AArch64::UZP1_ZZZ_D, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7219 }
7220 return 0;
7221}
7222
7223unsigned fastEmit_AArch64ISD_UZP1_MVT_nxv4f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7224 if (RetVT.SimpleTy != MVT::nxv4f16)
7225 return 0;
7226 if ((Subtarget->hasSVE())) {
7227 return fastEmitInst_rr(AArch64::UZP1_ZZZ_S, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7228 }
7229 return 0;
7230}
7231
7232unsigned fastEmit_AArch64ISD_UZP1_MVT_nxv8f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7233 if (RetVT.SimpleTy != MVT::nxv8f16)
7234 return 0;
7235 if ((Subtarget->hasSVE())) {
7236 return fastEmitInst_rr(AArch64::UZP1_ZZZ_H, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7237 }
7238 return 0;
7239}
7240
7241unsigned fastEmit_AArch64ISD_UZP1_MVT_nxv8bf16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7242 if (RetVT.SimpleTy != MVT::nxv8bf16)
7243 return 0;
7244 if ((Subtarget->hasSVE())) {
7245 return fastEmitInst_rr(AArch64::UZP1_ZZZ_H, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7246 }
7247 return 0;
7248}
7249
7250unsigned fastEmit_AArch64ISD_UZP1_MVT_nxv2f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7251 if (RetVT.SimpleTy != MVT::nxv2f32)
7252 return 0;
7253 if ((Subtarget->hasSVE())) {
7254 return fastEmitInst_rr(AArch64::UZP1_ZZZ_D, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7255 }
7256 return 0;
7257}
7258
7259unsigned fastEmit_AArch64ISD_UZP1_MVT_nxv4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7260 if (RetVT.SimpleTy != MVT::nxv4f32)
7261 return 0;
7262 if ((Subtarget->hasSVE())) {
7263 return fastEmitInst_rr(AArch64::UZP1_ZZZ_S, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7264 }
7265 return 0;
7266}
7267
7268unsigned fastEmit_AArch64ISD_UZP1_MVT_nxv2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7269 if (RetVT.SimpleTy != MVT::nxv2f64)
7270 return 0;
7271 if ((Subtarget->hasSVE())) {
7272 return fastEmitInst_rr(AArch64::UZP1_ZZZ_D, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7273 }
7274 return 0;
7275}
7276
7277unsigned fastEmit_AArch64ISD_UZP1_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7278 switch (VT.SimpleTy) {
7279 case MVT::v8i8: return fastEmit_AArch64ISD_UZP1_MVT_v8i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7280 case MVT::v16i8: return fastEmit_AArch64ISD_UZP1_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7281 case MVT::v4i16: return fastEmit_AArch64ISD_UZP1_MVT_v4i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7282 case MVT::v8i16: return fastEmit_AArch64ISD_UZP1_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7283 case MVT::v2i32: return fastEmit_AArch64ISD_UZP1_MVT_v2i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7284 case MVT::v4i32: return fastEmit_AArch64ISD_UZP1_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7285 case MVT::v2i64: return fastEmit_AArch64ISD_UZP1_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7286 case MVT::v4f16: return fastEmit_AArch64ISD_UZP1_MVT_v4f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7287 case MVT::v8f16: return fastEmit_AArch64ISD_UZP1_MVT_v8f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7288 case MVT::v2f32: return fastEmit_AArch64ISD_UZP1_MVT_v2f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7289 case MVT::v4f32: return fastEmit_AArch64ISD_UZP1_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7290 case MVT::v2f64: return fastEmit_AArch64ISD_UZP1_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7291 case MVT::nxv2i1: return fastEmit_AArch64ISD_UZP1_MVT_nxv2i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7292 case MVT::nxv4i1: return fastEmit_AArch64ISD_UZP1_MVT_nxv4i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7293 case MVT::nxv8i1: return fastEmit_AArch64ISD_UZP1_MVT_nxv8i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7294 case MVT::nxv16i1: return fastEmit_AArch64ISD_UZP1_MVT_nxv16i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7295 case MVT::nxv16i8: return fastEmit_AArch64ISD_UZP1_MVT_nxv16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7296 case MVT::nxv8i16: return fastEmit_AArch64ISD_UZP1_MVT_nxv8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7297 case MVT::nxv4i32: return fastEmit_AArch64ISD_UZP1_MVT_nxv4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7298 case MVT::nxv2i64: return fastEmit_AArch64ISD_UZP1_MVT_nxv2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7299 case MVT::nxv2f16: return fastEmit_AArch64ISD_UZP1_MVT_nxv2f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7300 case MVT::nxv4f16: return fastEmit_AArch64ISD_UZP1_MVT_nxv4f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7301 case MVT::nxv8f16: return fastEmit_AArch64ISD_UZP1_MVT_nxv8f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7302 case MVT::nxv8bf16: return fastEmit_AArch64ISD_UZP1_MVT_nxv8bf16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7303 case MVT::nxv2f32: return fastEmit_AArch64ISD_UZP1_MVT_nxv2f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7304 case MVT::nxv4f32: return fastEmit_AArch64ISD_UZP1_MVT_nxv4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7305 case MVT::nxv2f64: return fastEmit_AArch64ISD_UZP1_MVT_nxv2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7306 default: return 0;
7307 }
7308}
7309
7310// FastEmit functions for AArch64ISD::UZP2.
7311
7312unsigned fastEmit_AArch64ISD_UZP2_MVT_v8i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7313 if (RetVT.SimpleTy != MVT::v8i8)
7314 return 0;
7315 if ((Subtarget->hasNEON())) {
7316 return fastEmitInst_rr(AArch64::UZP2v8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7317 }
7318 return 0;
7319}
7320
7321unsigned fastEmit_AArch64ISD_UZP2_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7322 if (RetVT.SimpleTy != MVT::v16i8)
7323 return 0;
7324 if ((Subtarget->hasNEON())) {
7325 return fastEmitInst_rr(AArch64::UZP2v16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7326 }
7327 return 0;
7328}
7329
7330unsigned fastEmit_AArch64ISD_UZP2_MVT_v4i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7331 if (RetVT.SimpleTy != MVT::v4i16)
7332 return 0;
7333 if ((Subtarget->hasNEON())) {
7334 return fastEmitInst_rr(AArch64::UZP2v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7335 }
7336 return 0;
7337}
7338
7339unsigned fastEmit_AArch64ISD_UZP2_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7340 if (RetVT.SimpleTy != MVT::v8i16)
7341 return 0;
7342 if ((Subtarget->hasNEON())) {
7343 return fastEmitInst_rr(AArch64::UZP2v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7344 }
7345 return 0;
7346}
7347
7348unsigned fastEmit_AArch64ISD_UZP2_MVT_v2i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7349 if (RetVT.SimpleTy != MVT::v2i32)
7350 return 0;
7351 if ((Subtarget->hasNEON())) {
7352 return fastEmitInst_rr(AArch64::UZP2v2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7353 }
7354 return 0;
7355}
7356
7357unsigned fastEmit_AArch64ISD_UZP2_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7358 if (RetVT.SimpleTy != MVT::v4i32)
7359 return 0;
7360 if ((Subtarget->hasNEON())) {
7361 return fastEmitInst_rr(AArch64::UZP2v4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7362 }
7363 return 0;
7364}
7365
7366unsigned fastEmit_AArch64ISD_UZP2_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7367 if (RetVT.SimpleTy != MVT::v2i64)
7368 return 0;
7369 if ((Subtarget->hasNEON())) {
7370 return fastEmitInst_rr(AArch64::UZP2v2i64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7371 }
7372 return 0;
7373}
7374
7375unsigned fastEmit_AArch64ISD_UZP2_MVT_v4f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7376 if (RetVT.SimpleTy != MVT::v4f16)
7377 return 0;
7378 if ((Subtarget->hasNEON())) {
7379 return fastEmitInst_rr(AArch64::UZP2v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7380 }
7381 return 0;
7382}
7383
7384unsigned fastEmit_AArch64ISD_UZP2_MVT_v8f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7385 if (RetVT.SimpleTy != MVT::v8f16)
7386 return 0;
7387 if ((Subtarget->hasNEON())) {
7388 return fastEmitInst_rr(AArch64::UZP2v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7389 }
7390 return 0;
7391}
7392
7393unsigned fastEmit_AArch64ISD_UZP2_MVT_v2f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7394 if (RetVT.SimpleTy != MVT::v2f32)
7395 return 0;
7396 if ((Subtarget->hasNEON())) {
7397 return fastEmitInst_rr(AArch64::UZP2v2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7398 }
7399 return 0;
7400}
7401
7402unsigned fastEmit_AArch64ISD_UZP2_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7403 if (RetVT.SimpleTy != MVT::v4f32)
7404 return 0;
7405 if ((Subtarget->hasNEON())) {
7406 return fastEmitInst_rr(AArch64::UZP2v4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7407 }
7408 return 0;
7409}
7410
7411unsigned fastEmit_AArch64ISD_UZP2_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7412 if (RetVT.SimpleTy != MVT::v2f64)
7413 return 0;
7414 if ((Subtarget->hasNEON())) {
7415 return fastEmitInst_rr(AArch64::UZP2v2i64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7416 }
7417 return 0;
7418}
7419
7420unsigned fastEmit_AArch64ISD_UZP2_MVT_nxv2i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7421 if (RetVT.SimpleTy != MVT::nxv2i1)
7422 return 0;
7423 if ((Subtarget->hasSVE())) {
7424 return fastEmitInst_rr(AArch64::UZP2_PPP_D, &AArch64::PPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7425 }
7426 return 0;
7427}
7428
7429unsigned fastEmit_AArch64ISD_UZP2_MVT_nxv4i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7430 if (RetVT.SimpleTy != MVT::nxv4i1)
7431 return 0;
7432 if ((Subtarget->hasSVE())) {
7433 return fastEmitInst_rr(AArch64::UZP2_PPP_S, &AArch64::PPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7434 }
7435 return 0;
7436}
7437
7438unsigned fastEmit_AArch64ISD_UZP2_MVT_nxv8i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7439 if (RetVT.SimpleTy != MVT::nxv8i1)
7440 return 0;
7441 if ((Subtarget->hasSVE())) {
7442 return fastEmitInst_rr(AArch64::UZP2_PPP_H, &AArch64::PPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7443 }
7444 return 0;
7445}
7446
7447unsigned fastEmit_AArch64ISD_UZP2_MVT_nxv16i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7448 if (RetVT.SimpleTy != MVT::nxv16i1)
7449 return 0;
7450 if ((Subtarget->hasSVE())) {
7451 return fastEmitInst_rr(AArch64::UZP2_PPP_B, &AArch64::PPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7452 }
7453 return 0;
7454}
7455
7456unsigned fastEmit_AArch64ISD_UZP2_MVT_nxv16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7457 if (RetVT.SimpleTy != MVT::nxv16i8)
7458 return 0;
7459 if ((Subtarget->hasSVE())) {
7460 return fastEmitInst_rr(AArch64::UZP2_ZZZ_B, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7461 }
7462 return 0;
7463}
7464
7465unsigned fastEmit_AArch64ISD_UZP2_MVT_nxv8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7466 if (RetVT.SimpleTy != MVT::nxv8i16)
7467 return 0;
7468 if ((Subtarget->hasSVE())) {
7469 return fastEmitInst_rr(AArch64::UZP2_ZZZ_H, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7470 }
7471 return 0;
7472}
7473
7474unsigned fastEmit_AArch64ISD_UZP2_MVT_nxv4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7475 if (RetVT.SimpleTy != MVT::nxv4i32)
7476 return 0;
7477 if ((Subtarget->hasSVE())) {
7478 return fastEmitInst_rr(AArch64::UZP2_ZZZ_S, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7479 }
7480 return 0;
7481}
7482
7483unsigned fastEmit_AArch64ISD_UZP2_MVT_nxv2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7484 if (RetVT.SimpleTy != MVT::nxv2i64)
7485 return 0;
7486 if ((Subtarget->hasSVE())) {
7487 return fastEmitInst_rr(AArch64::UZP2_ZZZ_D, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7488 }
7489 return 0;
7490}
7491
7492unsigned fastEmit_AArch64ISD_UZP2_MVT_nxv2f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7493 if (RetVT.SimpleTy != MVT::nxv2f16)
7494 return 0;
7495 if ((Subtarget->hasSVE())) {
7496 return fastEmitInst_rr(AArch64::UZP2_ZZZ_D, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7497 }
7498 return 0;
7499}
7500
7501unsigned fastEmit_AArch64ISD_UZP2_MVT_nxv4f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7502 if (RetVT.SimpleTy != MVT::nxv4f16)
7503 return 0;
7504 if ((Subtarget->hasSVE())) {
7505 return fastEmitInst_rr(AArch64::UZP2_ZZZ_S, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7506 }
7507 return 0;
7508}
7509
7510unsigned fastEmit_AArch64ISD_UZP2_MVT_nxv8f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7511 if (RetVT.SimpleTy != MVT::nxv8f16)
7512 return 0;
7513 if ((Subtarget->hasSVE())) {
7514 return fastEmitInst_rr(AArch64::UZP2_ZZZ_H, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7515 }
7516 return 0;
7517}
7518
7519unsigned fastEmit_AArch64ISD_UZP2_MVT_nxv8bf16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7520 if (RetVT.SimpleTy != MVT::nxv8bf16)
7521 return 0;
7522 if ((Subtarget->hasSVE())) {
7523 return fastEmitInst_rr(AArch64::UZP2_ZZZ_H, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7524 }
7525 return 0;
7526}
7527
7528unsigned fastEmit_AArch64ISD_UZP2_MVT_nxv2f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7529 if (RetVT.SimpleTy != MVT::nxv2f32)
7530 return 0;
7531 if ((Subtarget->hasSVE())) {
7532 return fastEmitInst_rr(AArch64::UZP2_ZZZ_D, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7533 }
7534 return 0;
7535}
7536
7537unsigned fastEmit_AArch64ISD_UZP2_MVT_nxv4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7538 if (RetVT.SimpleTy != MVT::nxv4f32)
7539 return 0;
7540 if ((Subtarget->hasSVE())) {
7541 return fastEmitInst_rr(AArch64::UZP2_ZZZ_S, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7542 }
7543 return 0;
7544}
7545
7546unsigned fastEmit_AArch64ISD_UZP2_MVT_nxv2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7547 if (RetVT.SimpleTy != MVT::nxv2f64)
7548 return 0;
7549 if ((Subtarget->hasSVE())) {
7550 return fastEmitInst_rr(AArch64::UZP2_ZZZ_D, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7551 }
7552 return 0;
7553}
7554
7555unsigned fastEmit_AArch64ISD_UZP2_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7556 switch (VT.SimpleTy) {
7557 case MVT::v8i8: return fastEmit_AArch64ISD_UZP2_MVT_v8i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7558 case MVT::v16i8: return fastEmit_AArch64ISD_UZP2_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7559 case MVT::v4i16: return fastEmit_AArch64ISD_UZP2_MVT_v4i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7560 case MVT::v8i16: return fastEmit_AArch64ISD_UZP2_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7561 case MVT::v2i32: return fastEmit_AArch64ISD_UZP2_MVT_v2i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7562 case MVT::v4i32: return fastEmit_AArch64ISD_UZP2_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7563 case MVT::v2i64: return fastEmit_AArch64ISD_UZP2_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7564 case MVT::v4f16: return fastEmit_AArch64ISD_UZP2_MVT_v4f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7565 case MVT::v8f16: return fastEmit_AArch64ISD_UZP2_MVT_v8f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7566 case MVT::v2f32: return fastEmit_AArch64ISD_UZP2_MVT_v2f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7567 case MVT::v4f32: return fastEmit_AArch64ISD_UZP2_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7568 case MVT::v2f64: return fastEmit_AArch64ISD_UZP2_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7569 case MVT::nxv2i1: return fastEmit_AArch64ISD_UZP2_MVT_nxv2i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7570 case MVT::nxv4i1: return fastEmit_AArch64ISD_UZP2_MVT_nxv4i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7571 case MVT::nxv8i1: return fastEmit_AArch64ISD_UZP2_MVT_nxv8i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7572 case MVT::nxv16i1: return fastEmit_AArch64ISD_UZP2_MVT_nxv16i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7573 case MVT::nxv16i8: return fastEmit_AArch64ISD_UZP2_MVT_nxv16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7574 case MVT::nxv8i16: return fastEmit_AArch64ISD_UZP2_MVT_nxv8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7575 case MVT::nxv4i32: return fastEmit_AArch64ISD_UZP2_MVT_nxv4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7576 case MVT::nxv2i64: return fastEmit_AArch64ISD_UZP2_MVT_nxv2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7577 case MVT::nxv2f16: return fastEmit_AArch64ISD_UZP2_MVT_nxv2f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7578 case MVT::nxv4f16: return fastEmit_AArch64ISD_UZP2_MVT_nxv4f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7579 case MVT::nxv8f16: return fastEmit_AArch64ISD_UZP2_MVT_nxv8f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7580 case MVT::nxv8bf16: return fastEmit_AArch64ISD_UZP2_MVT_nxv8bf16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7581 case MVT::nxv2f32: return fastEmit_AArch64ISD_UZP2_MVT_nxv2f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7582 case MVT::nxv4f32: return fastEmit_AArch64ISD_UZP2_MVT_nxv4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7583 case MVT::nxv2f64: return fastEmit_AArch64ISD_UZP2_MVT_nxv2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7584 default: return 0;
7585 }
7586}
7587
7588// FastEmit functions for AArch64ISD::ZIP1.
7589
7590unsigned fastEmit_AArch64ISD_ZIP1_MVT_v8i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7591 if (RetVT.SimpleTy != MVT::v8i8)
7592 return 0;
7593 if ((Subtarget->hasNEON())) {
7594 return fastEmitInst_rr(AArch64::ZIP1v8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7595 }
7596 return 0;
7597}
7598
7599unsigned fastEmit_AArch64ISD_ZIP1_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7600 if (RetVT.SimpleTy != MVT::v16i8)
7601 return 0;
7602 if ((Subtarget->hasNEON())) {
7603 return fastEmitInst_rr(AArch64::ZIP1v16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7604 }
7605 return 0;
7606}
7607
7608unsigned fastEmit_AArch64ISD_ZIP1_MVT_v4i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7609 if (RetVT.SimpleTy != MVT::v4i16)
7610 return 0;
7611 if ((Subtarget->hasNEON())) {
7612 return fastEmitInst_rr(AArch64::ZIP1v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7613 }
7614 return 0;
7615}
7616
7617unsigned fastEmit_AArch64ISD_ZIP1_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7618 if (RetVT.SimpleTy != MVT::v8i16)
7619 return 0;
7620 if ((Subtarget->hasNEON())) {
7621 return fastEmitInst_rr(AArch64::ZIP1v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7622 }
7623 return 0;
7624}
7625
7626unsigned fastEmit_AArch64ISD_ZIP1_MVT_v2i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7627 if (RetVT.SimpleTy != MVT::v2i32)
7628 return 0;
7629 if ((Subtarget->hasNEON())) {
7630 return fastEmitInst_rr(AArch64::ZIP1v2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7631 }
7632 return 0;
7633}
7634
7635unsigned fastEmit_AArch64ISD_ZIP1_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7636 if (RetVT.SimpleTy != MVT::v4i32)
7637 return 0;
7638 if ((Subtarget->hasNEON())) {
7639 return fastEmitInst_rr(AArch64::ZIP1v4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7640 }
7641 return 0;
7642}
7643
7644unsigned fastEmit_AArch64ISD_ZIP1_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7645 if (RetVT.SimpleTy != MVT::v2i64)
7646 return 0;
7647 if ((Subtarget->hasNEON())) {
7648 return fastEmitInst_rr(AArch64::ZIP1v2i64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7649 }
7650 return 0;
7651}
7652
7653unsigned fastEmit_AArch64ISD_ZIP1_MVT_v4f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7654 if (RetVT.SimpleTy != MVT::v4f16)
7655 return 0;
7656 if ((Subtarget->hasNEON())) {
7657 return fastEmitInst_rr(AArch64::ZIP1v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7658 }
7659 return 0;
7660}
7661
7662unsigned fastEmit_AArch64ISD_ZIP1_MVT_v8f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7663 if (RetVT.SimpleTy != MVT::v8f16)
7664 return 0;
7665 if ((Subtarget->hasNEON())) {
7666 return fastEmitInst_rr(AArch64::ZIP1v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7667 }
7668 return 0;
7669}
7670
7671unsigned fastEmit_AArch64ISD_ZIP1_MVT_v2f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7672 if (RetVT.SimpleTy != MVT::v2f32)
7673 return 0;
7674 if ((Subtarget->hasNEON())) {
7675 return fastEmitInst_rr(AArch64::ZIP1v2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7676 }
7677 return 0;
7678}
7679
7680unsigned fastEmit_AArch64ISD_ZIP1_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7681 if (RetVT.SimpleTy != MVT::v4f32)
7682 return 0;
7683 if ((Subtarget->hasNEON())) {
7684 return fastEmitInst_rr(AArch64::ZIP1v4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7685 }
7686 return 0;
7687}
7688
7689unsigned fastEmit_AArch64ISD_ZIP1_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7690 if (RetVT.SimpleTy != MVT::v2f64)
7691 return 0;
7692 if ((Subtarget->hasNEON())) {
7693 return fastEmitInst_rr(AArch64::ZIP1v2i64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7694 }
7695 return 0;
7696}
7697
7698unsigned fastEmit_AArch64ISD_ZIP1_MVT_nxv2i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7699 if (RetVT.SimpleTy != MVT::nxv2i1)
7700 return 0;
7701 if ((Subtarget->hasSVE())) {
7702 return fastEmitInst_rr(AArch64::ZIP1_PPP_D, &AArch64::PPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7703 }
7704 return 0;
7705}
7706
7707unsigned fastEmit_AArch64ISD_ZIP1_MVT_nxv4i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7708 if (RetVT.SimpleTy != MVT::nxv4i1)
7709 return 0;
7710 if ((Subtarget->hasSVE())) {
7711 return fastEmitInst_rr(AArch64::ZIP1_PPP_S, &AArch64::PPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7712 }
7713 return 0;
7714}
7715
7716unsigned fastEmit_AArch64ISD_ZIP1_MVT_nxv8i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7717 if (RetVT.SimpleTy != MVT::nxv8i1)
7718 return 0;
7719 if ((Subtarget->hasSVE())) {
7720 return fastEmitInst_rr(AArch64::ZIP1_PPP_H, &AArch64::PPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7721 }
7722 return 0;
7723}
7724
7725unsigned fastEmit_AArch64ISD_ZIP1_MVT_nxv16i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7726 if (RetVT.SimpleTy != MVT::nxv16i1)
7727 return 0;
7728 if ((Subtarget->hasSVE())) {
7729 return fastEmitInst_rr(AArch64::ZIP1_PPP_B, &AArch64::PPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7730 }
7731 return 0;
7732}
7733
7734unsigned fastEmit_AArch64ISD_ZIP1_MVT_nxv16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7735 if (RetVT.SimpleTy != MVT::nxv16i8)
7736 return 0;
7737 if ((Subtarget->hasSVE())) {
7738 return fastEmitInst_rr(AArch64::ZIP1_ZZZ_B, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7739 }
7740 return 0;
7741}
7742
7743unsigned fastEmit_AArch64ISD_ZIP1_MVT_nxv8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7744 if (RetVT.SimpleTy != MVT::nxv8i16)
7745 return 0;
7746 if ((Subtarget->hasSVE())) {
7747 return fastEmitInst_rr(AArch64::ZIP1_ZZZ_H, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7748 }
7749 return 0;
7750}
7751
7752unsigned fastEmit_AArch64ISD_ZIP1_MVT_nxv4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7753 if (RetVT.SimpleTy != MVT::nxv4i32)
7754 return 0;
7755 if ((Subtarget->hasSVE())) {
7756 return fastEmitInst_rr(AArch64::ZIP1_ZZZ_S, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7757 }
7758 return 0;
7759}
7760
7761unsigned fastEmit_AArch64ISD_ZIP1_MVT_nxv2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7762 if (RetVT.SimpleTy != MVT::nxv2i64)
7763 return 0;
7764 if ((Subtarget->hasSVE())) {
7765 return fastEmitInst_rr(AArch64::ZIP1_ZZZ_D, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7766 }
7767 return 0;
7768}
7769
7770unsigned fastEmit_AArch64ISD_ZIP1_MVT_nxv2f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7771 if (RetVT.SimpleTy != MVT::nxv2f16)
7772 return 0;
7773 if ((Subtarget->hasSVE())) {
7774 return fastEmitInst_rr(AArch64::ZIP1_ZZZ_D, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7775 }
7776 return 0;
7777}
7778
7779unsigned fastEmit_AArch64ISD_ZIP1_MVT_nxv4f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7780 if (RetVT.SimpleTy != MVT::nxv4f16)
7781 return 0;
7782 if ((Subtarget->hasSVE())) {
7783 return fastEmitInst_rr(AArch64::ZIP1_ZZZ_S, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7784 }
7785 return 0;
7786}
7787
7788unsigned fastEmit_AArch64ISD_ZIP1_MVT_nxv8f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7789 if (RetVT.SimpleTy != MVT::nxv8f16)
7790 return 0;
7791 if ((Subtarget->hasSVE())) {
7792 return fastEmitInst_rr(AArch64::ZIP1_ZZZ_H, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7793 }
7794 return 0;
7795}
7796
7797unsigned fastEmit_AArch64ISD_ZIP1_MVT_nxv8bf16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7798 if (RetVT.SimpleTy != MVT::nxv8bf16)
7799 return 0;
7800 if ((Subtarget->hasSVE())) {
7801 return fastEmitInst_rr(AArch64::ZIP1_ZZZ_H, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7802 }
7803 return 0;
7804}
7805
7806unsigned fastEmit_AArch64ISD_ZIP1_MVT_nxv2f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7807 if (RetVT.SimpleTy != MVT::nxv2f32)
7808 return 0;
7809 if ((Subtarget->hasSVE())) {
7810 return fastEmitInst_rr(AArch64::ZIP1_ZZZ_D, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7811 }
7812 return 0;
7813}
7814
7815unsigned fastEmit_AArch64ISD_ZIP1_MVT_nxv4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7816 if (RetVT.SimpleTy != MVT::nxv4f32)
7817 return 0;
7818 if ((Subtarget->hasSVE())) {
7819 return fastEmitInst_rr(AArch64::ZIP1_ZZZ_S, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7820 }
7821 return 0;
7822}
7823
7824unsigned fastEmit_AArch64ISD_ZIP1_MVT_nxv2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7825 if (RetVT.SimpleTy != MVT::nxv2f64)
7826 return 0;
7827 if ((Subtarget->hasSVE())) {
7828 return fastEmitInst_rr(AArch64::ZIP1_ZZZ_D, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7829 }
7830 return 0;
7831}
7832
7833unsigned fastEmit_AArch64ISD_ZIP1_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7834 switch (VT.SimpleTy) {
7835 case MVT::v8i8: return fastEmit_AArch64ISD_ZIP1_MVT_v8i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7836 case MVT::v16i8: return fastEmit_AArch64ISD_ZIP1_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7837 case MVT::v4i16: return fastEmit_AArch64ISD_ZIP1_MVT_v4i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7838 case MVT::v8i16: return fastEmit_AArch64ISD_ZIP1_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7839 case MVT::v2i32: return fastEmit_AArch64ISD_ZIP1_MVT_v2i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7840 case MVT::v4i32: return fastEmit_AArch64ISD_ZIP1_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7841 case MVT::v2i64: return fastEmit_AArch64ISD_ZIP1_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7842 case MVT::v4f16: return fastEmit_AArch64ISD_ZIP1_MVT_v4f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7843 case MVT::v8f16: return fastEmit_AArch64ISD_ZIP1_MVT_v8f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7844 case MVT::v2f32: return fastEmit_AArch64ISD_ZIP1_MVT_v2f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7845 case MVT::v4f32: return fastEmit_AArch64ISD_ZIP1_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7846 case MVT::v2f64: return fastEmit_AArch64ISD_ZIP1_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7847 case MVT::nxv2i1: return fastEmit_AArch64ISD_ZIP1_MVT_nxv2i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7848 case MVT::nxv4i1: return fastEmit_AArch64ISD_ZIP1_MVT_nxv4i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7849 case MVT::nxv8i1: return fastEmit_AArch64ISD_ZIP1_MVT_nxv8i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7850 case MVT::nxv16i1: return fastEmit_AArch64ISD_ZIP1_MVT_nxv16i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7851 case MVT::nxv16i8: return fastEmit_AArch64ISD_ZIP1_MVT_nxv16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7852 case MVT::nxv8i16: return fastEmit_AArch64ISD_ZIP1_MVT_nxv8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7853 case MVT::nxv4i32: return fastEmit_AArch64ISD_ZIP1_MVT_nxv4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7854 case MVT::nxv2i64: return fastEmit_AArch64ISD_ZIP1_MVT_nxv2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7855 case MVT::nxv2f16: return fastEmit_AArch64ISD_ZIP1_MVT_nxv2f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7856 case MVT::nxv4f16: return fastEmit_AArch64ISD_ZIP1_MVT_nxv4f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7857 case MVT::nxv8f16: return fastEmit_AArch64ISD_ZIP1_MVT_nxv8f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7858 case MVT::nxv8bf16: return fastEmit_AArch64ISD_ZIP1_MVT_nxv8bf16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7859 case MVT::nxv2f32: return fastEmit_AArch64ISD_ZIP1_MVT_nxv2f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7860 case MVT::nxv4f32: return fastEmit_AArch64ISD_ZIP1_MVT_nxv4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7861 case MVT::nxv2f64: return fastEmit_AArch64ISD_ZIP1_MVT_nxv2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7862 default: return 0;
7863 }
7864}
7865
7866// FastEmit functions for AArch64ISD::ZIP2.
7867
7868unsigned fastEmit_AArch64ISD_ZIP2_MVT_v8i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7869 if (RetVT.SimpleTy != MVT::v8i8)
7870 return 0;
7871 if ((Subtarget->hasNEON())) {
7872 return fastEmitInst_rr(AArch64::ZIP2v8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7873 }
7874 return 0;
7875}
7876
7877unsigned fastEmit_AArch64ISD_ZIP2_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7878 if (RetVT.SimpleTy != MVT::v16i8)
7879 return 0;
7880 if ((Subtarget->hasNEON())) {
7881 return fastEmitInst_rr(AArch64::ZIP2v16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7882 }
7883 return 0;
7884}
7885
7886unsigned fastEmit_AArch64ISD_ZIP2_MVT_v4i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7887 if (RetVT.SimpleTy != MVT::v4i16)
7888 return 0;
7889 if ((Subtarget->hasNEON())) {
7890 return fastEmitInst_rr(AArch64::ZIP2v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7891 }
7892 return 0;
7893}
7894
7895unsigned fastEmit_AArch64ISD_ZIP2_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7896 if (RetVT.SimpleTy != MVT::v8i16)
7897 return 0;
7898 if ((Subtarget->hasNEON())) {
7899 return fastEmitInst_rr(AArch64::ZIP2v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7900 }
7901 return 0;
7902}
7903
7904unsigned fastEmit_AArch64ISD_ZIP2_MVT_v2i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7905 if (RetVT.SimpleTy != MVT::v2i32)
7906 return 0;
7907 if ((Subtarget->hasNEON())) {
7908 return fastEmitInst_rr(AArch64::ZIP2v2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7909 }
7910 return 0;
7911}
7912
7913unsigned fastEmit_AArch64ISD_ZIP2_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7914 if (RetVT.SimpleTy != MVT::v4i32)
7915 return 0;
7916 if ((Subtarget->hasNEON())) {
7917 return fastEmitInst_rr(AArch64::ZIP2v4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7918 }
7919 return 0;
7920}
7921
7922unsigned fastEmit_AArch64ISD_ZIP2_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7923 if (RetVT.SimpleTy != MVT::v2i64)
7924 return 0;
7925 if ((Subtarget->hasNEON())) {
7926 return fastEmitInst_rr(AArch64::ZIP2v2i64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7927 }
7928 return 0;
7929}
7930
7931unsigned fastEmit_AArch64ISD_ZIP2_MVT_v4f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7932 if (RetVT.SimpleTy != MVT::v4f16)
7933 return 0;
7934 if ((Subtarget->hasNEON())) {
7935 return fastEmitInst_rr(AArch64::ZIP2v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7936 }
7937 return 0;
7938}
7939
7940unsigned fastEmit_AArch64ISD_ZIP2_MVT_v8f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7941 if (RetVT.SimpleTy != MVT::v8f16)
7942 return 0;
7943 if ((Subtarget->hasNEON())) {
7944 return fastEmitInst_rr(AArch64::ZIP2v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7945 }
7946 return 0;
7947}
7948
7949unsigned fastEmit_AArch64ISD_ZIP2_MVT_v2f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7950 if (RetVT.SimpleTy != MVT::v2f32)
7951 return 0;
7952 if ((Subtarget->hasNEON())) {
7953 return fastEmitInst_rr(AArch64::ZIP2v2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7954 }
7955 return 0;
7956}
7957
7958unsigned fastEmit_AArch64ISD_ZIP2_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7959 if (RetVT.SimpleTy != MVT::v4f32)
7960 return 0;
7961 if ((Subtarget->hasNEON())) {
7962 return fastEmitInst_rr(AArch64::ZIP2v4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7963 }
7964 return 0;
7965}
7966
7967unsigned fastEmit_AArch64ISD_ZIP2_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7968 if (RetVT.SimpleTy != MVT::v2f64)
7969 return 0;
7970 if ((Subtarget->hasNEON())) {
7971 return fastEmitInst_rr(AArch64::ZIP2v2i64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7972 }
7973 return 0;
7974}
7975
7976unsigned fastEmit_AArch64ISD_ZIP2_MVT_nxv2i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7977 if (RetVT.SimpleTy != MVT::nxv2i1)
7978 return 0;
7979 if ((Subtarget->hasSVE())) {
7980 return fastEmitInst_rr(AArch64::ZIP2_PPP_D, &AArch64::PPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7981 }
7982 return 0;
7983}
7984
7985unsigned fastEmit_AArch64ISD_ZIP2_MVT_nxv4i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7986 if (RetVT.SimpleTy != MVT::nxv4i1)
7987 return 0;
7988 if ((Subtarget->hasSVE())) {
7989 return fastEmitInst_rr(AArch64::ZIP2_PPP_S, &AArch64::PPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7990 }
7991 return 0;
7992}
7993
7994unsigned fastEmit_AArch64ISD_ZIP2_MVT_nxv8i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7995 if (RetVT.SimpleTy != MVT::nxv8i1)
7996 return 0;
7997 if ((Subtarget->hasSVE())) {
7998 return fastEmitInst_rr(AArch64::ZIP2_PPP_H, &AArch64::PPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7999 }
8000 return 0;
8001}
8002
8003unsigned fastEmit_AArch64ISD_ZIP2_MVT_nxv16i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8004 if (RetVT.SimpleTy != MVT::nxv16i1)
8005 return 0;
8006 if ((Subtarget->hasSVE())) {
8007 return fastEmitInst_rr(AArch64::ZIP2_PPP_B, &AArch64::PPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8008 }
8009 return 0;
8010}
8011
8012unsigned fastEmit_AArch64ISD_ZIP2_MVT_nxv16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8013 if (RetVT.SimpleTy != MVT::nxv16i8)
8014 return 0;
8015 if ((Subtarget->hasSVE())) {
8016 return fastEmitInst_rr(AArch64::ZIP2_ZZZ_B, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8017 }
8018 return 0;
8019}
8020
8021unsigned fastEmit_AArch64ISD_ZIP2_MVT_nxv8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8022 if (RetVT.SimpleTy != MVT::nxv8i16)
8023 return 0;
8024 if ((Subtarget->hasSVE())) {
8025 return fastEmitInst_rr(AArch64::ZIP2_ZZZ_H, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8026 }
8027 return 0;
8028}
8029
8030unsigned fastEmit_AArch64ISD_ZIP2_MVT_nxv4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8031 if (RetVT.SimpleTy != MVT::nxv4i32)
8032 return 0;
8033 if ((Subtarget->hasSVE())) {
8034 return fastEmitInst_rr(AArch64::ZIP2_ZZZ_S, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8035 }
8036 return 0;
8037}
8038
8039unsigned fastEmit_AArch64ISD_ZIP2_MVT_nxv2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8040 if (RetVT.SimpleTy != MVT::nxv2i64)
8041 return 0;
8042 if ((Subtarget->hasSVE())) {
8043 return fastEmitInst_rr(AArch64::ZIP2_ZZZ_D, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8044 }
8045 return 0;
8046}
8047
8048unsigned fastEmit_AArch64ISD_ZIP2_MVT_nxv2f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8049 if (RetVT.SimpleTy != MVT::nxv2f16)
8050 return 0;
8051 if ((Subtarget->hasSVE())) {
8052 return fastEmitInst_rr(AArch64::ZIP2_ZZZ_D, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8053 }
8054 return 0;
8055}
8056
8057unsigned fastEmit_AArch64ISD_ZIP2_MVT_nxv4f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8058 if (RetVT.SimpleTy != MVT::nxv4f16)
8059 return 0;
8060 if ((Subtarget->hasSVE())) {
8061 return fastEmitInst_rr(AArch64::ZIP2_ZZZ_S, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8062 }
8063 return 0;
8064}
8065
8066unsigned fastEmit_AArch64ISD_ZIP2_MVT_nxv8f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8067 if (RetVT.SimpleTy != MVT::nxv8f16)
8068 return 0;
8069 if ((Subtarget->hasSVE())) {
8070 return fastEmitInst_rr(AArch64::ZIP2_ZZZ_H, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8071 }
8072 return 0;
8073}
8074
8075unsigned fastEmit_AArch64ISD_ZIP2_MVT_nxv8bf16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8076 if (RetVT.SimpleTy != MVT::nxv8bf16)
8077 return 0;
8078 if ((Subtarget->hasSVE())) {
8079 return fastEmitInst_rr(AArch64::ZIP2_ZZZ_H, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8080 }
8081 return 0;
8082}
8083
8084unsigned fastEmit_AArch64ISD_ZIP2_MVT_nxv2f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8085 if (RetVT.SimpleTy != MVT::nxv2f32)
8086 return 0;
8087 if ((Subtarget->hasSVE())) {
8088 return fastEmitInst_rr(AArch64::ZIP2_ZZZ_D, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8089 }
8090 return 0;
8091}
8092
8093unsigned fastEmit_AArch64ISD_ZIP2_MVT_nxv4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8094 if (RetVT.SimpleTy != MVT::nxv4f32)
8095 return 0;
8096 if ((Subtarget->hasSVE())) {
8097 return fastEmitInst_rr(AArch64::ZIP2_ZZZ_S, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8098 }
8099 return 0;
8100}
8101
8102unsigned fastEmit_AArch64ISD_ZIP2_MVT_nxv2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8103 if (RetVT.SimpleTy != MVT::nxv2f64)
8104 return 0;
8105 if ((Subtarget->hasSVE())) {
8106 return fastEmitInst_rr(AArch64::ZIP2_ZZZ_D, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8107 }
8108 return 0;
8109}
8110
8111unsigned fastEmit_AArch64ISD_ZIP2_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8112 switch (VT.SimpleTy) {
8113 case MVT::v8i8: return fastEmit_AArch64ISD_ZIP2_MVT_v8i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8114 case MVT::v16i8: return fastEmit_AArch64ISD_ZIP2_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8115 case MVT::v4i16: return fastEmit_AArch64ISD_ZIP2_MVT_v4i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8116 case MVT::v8i16: return fastEmit_AArch64ISD_ZIP2_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8117 case MVT::v2i32: return fastEmit_AArch64ISD_ZIP2_MVT_v2i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8118 case MVT::v4i32: return fastEmit_AArch64ISD_ZIP2_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8119 case MVT::v2i64: return fastEmit_AArch64ISD_ZIP2_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8120 case MVT::v4f16: return fastEmit_AArch64ISD_ZIP2_MVT_v4f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8121 case MVT::v8f16: return fastEmit_AArch64ISD_ZIP2_MVT_v8f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8122 case MVT::v2f32: return fastEmit_AArch64ISD_ZIP2_MVT_v2f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8123 case MVT::v4f32: return fastEmit_AArch64ISD_ZIP2_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8124 case MVT::v2f64: return fastEmit_AArch64ISD_ZIP2_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8125 case MVT::nxv2i1: return fastEmit_AArch64ISD_ZIP2_MVT_nxv2i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8126 case MVT::nxv4i1: return fastEmit_AArch64ISD_ZIP2_MVT_nxv4i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8127 case MVT::nxv8i1: return fastEmit_AArch64ISD_ZIP2_MVT_nxv8i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8128 case MVT::nxv16i1: return fastEmit_AArch64ISD_ZIP2_MVT_nxv16i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8129 case MVT::nxv16i8: return fastEmit_AArch64ISD_ZIP2_MVT_nxv16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8130 case MVT::nxv8i16: return fastEmit_AArch64ISD_ZIP2_MVT_nxv8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8131 case MVT::nxv4i32: return fastEmit_AArch64ISD_ZIP2_MVT_nxv4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8132 case MVT::nxv2i64: return fastEmit_AArch64ISD_ZIP2_MVT_nxv2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8133 case MVT::nxv2f16: return fastEmit_AArch64ISD_ZIP2_MVT_nxv2f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8134 case MVT::nxv4f16: return fastEmit_AArch64ISD_ZIP2_MVT_nxv4f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8135 case MVT::nxv8f16: return fastEmit_AArch64ISD_ZIP2_MVT_nxv8f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8136 case MVT::nxv8bf16: return fastEmit_AArch64ISD_ZIP2_MVT_nxv8bf16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8137 case MVT::nxv2f32: return fastEmit_AArch64ISD_ZIP2_MVT_nxv2f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8138 case MVT::nxv4f32: return fastEmit_AArch64ISD_ZIP2_MVT_nxv4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8139 case MVT::nxv2f64: return fastEmit_AArch64ISD_ZIP2_MVT_nxv2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8140 default: return 0;
8141 }
8142}
8143
8144// FastEmit functions for ISD::ADD.
8145
8146unsigned fastEmit_ISD_ADD_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8147 if (RetVT.SimpleTy != MVT::i32)
8148 return 0;
8149 return fastEmitInst_rr(AArch64::ADDWrr, &AArch64::GPR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8150}
8151
8152unsigned fastEmit_ISD_ADD_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8153 if (RetVT.SimpleTy != MVT::i64)
8154 return 0;
8155 return fastEmitInst_rr(AArch64::ADDXrr, &AArch64::GPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8156}
8157
8158unsigned fastEmit_ISD_ADD_MVT_v8i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8159 if (RetVT.SimpleTy != MVT::v8i8)
8160 return 0;
8161 if ((Subtarget->hasNEON())) {
8162 return fastEmitInst_rr(AArch64::ADDv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8163 }
8164 return 0;
8165}
8166
8167unsigned fastEmit_ISD_ADD_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8168 if (RetVT.SimpleTy != MVT::v16i8)
8169 return 0;
8170 if ((Subtarget->hasNEON())) {
8171 return fastEmitInst_rr(AArch64::ADDv16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8172 }
8173 return 0;
8174}
8175
8176unsigned fastEmit_ISD_ADD_MVT_v4i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8177 if (RetVT.SimpleTy != MVT::v4i16)
8178 return 0;
8179 if ((Subtarget->hasNEON())) {
8180 return fastEmitInst_rr(AArch64::ADDv4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8181 }
8182 return 0;
8183}
8184
8185unsigned fastEmit_ISD_ADD_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8186 if (RetVT.SimpleTy != MVT::v8i16)
8187 return 0;
8188 if ((Subtarget->hasNEON())) {
8189 return fastEmitInst_rr(AArch64::ADDv8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8190 }
8191 return 0;
8192}
8193
8194unsigned fastEmit_ISD_ADD_MVT_v2i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8195 if (RetVT.SimpleTy != MVT::v2i32)
8196 return 0;
8197 if ((Subtarget->hasNEON())) {
8198 return fastEmitInst_rr(AArch64::ADDv2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8199 }
8200 return 0;
8201}
8202
8203unsigned fastEmit_ISD_ADD_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8204 if (RetVT.SimpleTy != MVT::v4i32)
8205 return 0;
8206 if ((Subtarget->hasNEON())) {
8207 return fastEmitInst_rr(AArch64::ADDv4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8208 }
8209 return 0;
8210}
8211
8212unsigned fastEmit_ISD_ADD_MVT_v1i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8213 if (RetVT.SimpleTy != MVT::v1i64)
8214 return 0;
8215 if ((Subtarget->hasNEON())) {
8216 return fastEmitInst_rr(AArch64::ADDv1i64, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8217 }
8218 return 0;
8219}
8220
8221unsigned fastEmit_ISD_ADD_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8222 if (RetVT.SimpleTy != MVT::v2i64)
8223 return 0;
8224 if ((Subtarget->hasNEON())) {
8225 return fastEmitInst_rr(AArch64::ADDv2i64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8226 }
8227 return 0;
8228}
8229
8230unsigned fastEmit_ISD_ADD_MVT_nxv16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8231 if (RetVT.SimpleTy != MVT::nxv16i8)
8232 return 0;
8233 if ((Subtarget->hasSVE())) {
8234 return fastEmitInst_rr(AArch64::ADD_ZZZ_B, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8235 }
8236 return 0;
8237}
8238
8239unsigned fastEmit_ISD_ADD_MVT_nxv8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8240 if (RetVT.SimpleTy != MVT::nxv8i16)
8241 return 0;
8242 if ((Subtarget->hasSVE())) {
8243 return fastEmitInst_rr(AArch64::ADD_ZZZ_H, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8244 }
8245 return 0;
8246}
8247
8248unsigned fastEmit_ISD_ADD_MVT_nxv4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8249 if (RetVT.SimpleTy != MVT::nxv4i32)
8250 return 0;
8251 if ((Subtarget->hasSVE())) {
8252 return fastEmitInst_rr(AArch64::ADD_ZZZ_S, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8253 }
8254 return 0;
8255}
8256
8257unsigned fastEmit_ISD_ADD_MVT_nxv2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8258 if (RetVT.SimpleTy != MVT::nxv2i64)
8259 return 0;
8260 if ((Subtarget->hasSVE())) {
8261 return fastEmitInst_rr(AArch64::ADD_ZZZ_D, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8262 }
8263 return 0;
8264}
8265
8266unsigned fastEmit_ISD_ADD_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8267 switch (VT.SimpleTy) {
8268 case MVT::i32: return fastEmit_ISD_ADD_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8269 case MVT::i64: return fastEmit_ISD_ADD_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8270 case MVT::v8i8: return fastEmit_ISD_ADD_MVT_v8i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8271 case MVT::v16i8: return fastEmit_ISD_ADD_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8272 case MVT::v4i16: return fastEmit_ISD_ADD_MVT_v4i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8273 case MVT::v8i16: return fastEmit_ISD_ADD_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8274 case MVT::v2i32: return fastEmit_ISD_ADD_MVT_v2i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8275 case MVT::v4i32: return fastEmit_ISD_ADD_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8276 case MVT::v1i64: return fastEmit_ISD_ADD_MVT_v1i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8277 case MVT::v2i64: return fastEmit_ISD_ADD_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8278 case MVT::nxv16i8: return fastEmit_ISD_ADD_MVT_nxv16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8279 case MVT::nxv8i16: return fastEmit_ISD_ADD_MVT_nxv8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8280 case MVT::nxv4i32: return fastEmit_ISD_ADD_MVT_nxv4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8281 case MVT::nxv2i64: return fastEmit_ISD_ADD_MVT_nxv2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8282 default: return 0;
8283 }
8284}
8285
8286// FastEmit functions for ISD::AND.
8287
8288unsigned fastEmit_ISD_AND_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8289 if (RetVT.SimpleTy != MVT::i32)
8290 return 0;
8291 return fastEmitInst_rr(AArch64::ANDWrr, &AArch64::GPR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8292}
8293
8294unsigned fastEmit_ISD_AND_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8295 if (RetVT.SimpleTy != MVT::i64)
8296 return 0;
8297 return fastEmitInst_rr(AArch64::ANDXrr, &AArch64::GPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8298}
8299
8300unsigned fastEmit_ISD_AND_MVT_v8i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8301 if (RetVT.SimpleTy != MVT::v8i8)
8302 return 0;
8303 if ((Subtarget->hasNEON())) {
8304 return fastEmitInst_rr(AArch64::ANDv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8305 }
8306 return 0;
8307}
8308
8309unsigned fastEmit_ISD_AND_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8310 if (RetVT.SimpleTy != MVT::v16i8)
8311 return 0;
8312 if ((Subtarget->hasNEON())) {
8313 return fastEmitInst_rr(AArch64::ANDv16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8314 }
8315 return 0;
8316}
8317
8318unsigned fastEmit_ISD_AND_MVT_v4i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8319 if (RetVT.SimpleTy != MVT::v4i16)
8320 return 0;
8321 if ((Subtarget->hasNEON())) {
8322 return fastEmitInst_rr(AArch64::ANDv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8323 }
8324 return 0;
8325}
8326
8327unsigned fastEmit_ISD_AND_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8328 if (RetVT.SimpleTy != MVT::v8i16)
8329 return 0;
8330 if ((Subtarget->hasNEON())) {
8331 return fastEmitInst_rr(AArch64::ANDv16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8332 }
8333 return 0;
8334}
8335
8336unsigned fastEmit_ISD_AND_MVT_v2i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8337 if (RetVT.SimpleTy != MVT::v2i32)
8338 return 0;
8339 if ((Subtarget->hasNEON())) {
8340 return fastEmitInst_rr(AArch64::ANDv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8341 }
8342 return 0;
8343}
8344
8345unsigned fastEmit_ISD_AND_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8346 if (RetVT.SimpleTy != MVT::v4i32)
8347 return 0;
8348 if ((Subtarget->hasNEON())) {
8349 return fastEmitInst_rr(AArch64::ANDv16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8350 }
8351 return 0;
8352}
8353
8354unsigned fastEmit_ISD_AND_MVT_v1i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8355 if (RetVT.SimpleTy != MVT::v1i64)
8356 return 0;
8357 if ((Subtarget->hasNEON())) {
8358 return fastEmitInst_rr(AArch64::ANDv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8359 }
8360 return 0;
8361}
8362
8363unsigned fastEmit_ISD_AND_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8364 if (RetVT.SimpleTy != MVT::v2i64)
8365 return 0;
8366 if ((Subtarget->hasNEON())) {
8367 return fastEmitInst_rr(AArch64::ANDv16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8368 }
8369 return 0;
8370}
8371
8372unsigned fastEmit_ISD_AND_MVT_nxv16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8373 if (RetVT.SimpleTy != MVT::nxv16i8)
8374 return 0;
8375 if ((Subtarget->hasSVE())) {
8376 return fastEmitInst_rr(AArch64::AND_ZZZ, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8377 }
8378 return 0;
8379}
8380
8381unsigned fastEmit_ISD_AND_MVT_nxv8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8382 if (RetVT.SimpleTy != MVT::nxv8i16)
8383 return 0;
8384 if ((Subtarget->hasSVE())) {
8385 return fastEmitInst_rr(AArch64::AND_ZZZ, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8386 }
8387 return 0;
8388}
8389
8390unsigned fastEmit_ISD_AND_MVT_nxv4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8391 if (RetVT.SimpleTy != MVT::nxv4i32)
8392 return 0;
8393 if ((Subtarget->hasSVE())) {
8394 return fastEmitInst_rr(AArch64::AND_ZZZ, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8395 }
8396 return 0;
8397}
8398
8399unsigned fastEmit_ISD_AND_MVT_nxv2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8400 if (RetVT.SimpleTy != MVT::nxv2i64)
8401 return 0;
8402 if ((Subtarget->hasSVE())) {
8403 return fastEmitInst_rr(AArch64::AND_ZZZ, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8404 }
8405 return 0;
8406}
8407
8408unsigned fastEmit_ISD_AND_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8409 switch (VT.SimpleTy) {
8410 case MVT::i32: return fastEmit_ISD_AND_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8411 case MVT::i64: return fastEmit_ISD_AND_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8412 case MVT::v8i8: return fastEmit_ISD_AND_MVT_v8i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8413 case MVT::v16i8: return fastEmit_ISD_AND_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8414 case MVT::v4i16: return fastEmit_ISD_AND_MVT_v4i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8415 case MVT::v8i16: return fastEmit_ISD_AND_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8416 case MVT::v2i32: return fastEmit_ISD_AND_MVT_v2i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8417 case MVT::v4i32: return fastEmit_ISD_AND_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8418 case MVT::v1i64: return fastEmit_ISD_AND_MVT_v1i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8419 case MVT::v2i64: return fastEmit_ISD_AND_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8420 case MVT::nxv16i8: return fastEmit_ISD_AND_MVT_nxv16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8421 case MVT::nxv8i16: return fastEmit_ISD_AND_MVT_nxv8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8422 case MVT::nxv4i32: return fastEmit_ISD_AND_MVT_nxv4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8423 case MVT::nxv2i64: return fastEmit_ISD_AND_MVT_nxv2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8424 default: return 0;
8425 }
8426}
8427
8428// FastEmit functions for ISD::CONCAT_VECTORS.
8429
8430unsigned fastEmit_ISD_CONCAT_VECTORS_MVT_nxv2i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8431 if (RetVT.SimpleTy != MVT::nxv4i1)
8432 return 0;
8433 if ((Subtarget->hasSVE())) {
8434 return fastEmitInst_rr(AArch64::UZP1_PPP_S, &AArch64::PPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8435 }
8436 return 0;
8437}
8438
8439unsigned fastEmit_ISD_CONCAT_VECTORS_MVT_nxv4i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8440 if (RetVT.SimpleTy != MVT::nxv8i1)
8441 return 0;
8442 if ((Subtarget->hasSVE())) {
8443 return fastEmitInst_rr(AArch64::UZP1_PPP_H, &AArch64::PPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8444 }
8445 return 0;
8446}
8447
8448unsigned fastEmit_ISD_CONCAT_VECTORS_MVT_nxv8i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8449 if (RetVT.SimpleTy != MVT::nxv16i1)
8450 return 0;
8451 if ((Subtarget->hasSVE())) {
8452 return fastEmitInst_rr(AArch64::UZP1_PPP_B, &AArch64::PPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8453 }
8454 return 0;
8455}
8456
8457unsigned fastEmit_ISD_CONCAT_VECTORS_MVT_nxv2f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8458 if (RetVT.SimpleTy != MVT::nxv4f16)
8459 return 0;
8460 if ((Subtarget->hasSVE())) {
8461 return fastEmitInst_rr(AArch64::UZP1_ZZZ_S, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8462 }
8463 return 0;
8464}
8465
8466unsigned fastEmit_ISD_CONCAT_VECTORS_MVT_nxv4f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8467 if (RetVT.SimpleTy != MVT::nxv8f16)
8468 return 0;
8469 if ((Subtarget->hasSVE())) {
8470 return fastEmitInst_rr(AArch64::UZP1_ZZZ_H, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8471 }
8472 return 0;
8473}
8474
8475unsigned fastEmit_ISD_CONCAT_VECTORS_MVT_nxv2bf16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8476 if (RetVT.SimpleTy != MVT::nxv4bf16)
8477 return 0;
8478 if ((Subtarget->hasSVE())) {
8479 return fastEmitInst_rr(AArch64::UZP1_ZZZ_S, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8480 }
8481 return 0;
8482}
8483
8484unsigned fastEmit_ISD_CONCAT_VECTORS_MVT_nxv4bf16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8485 if (RetVT.SimpleTy != MVT::nxv8bf16)
8486 return 0;
8487 if ((Subtarget->hasSVE())) {
8488 return fastEmitInst_rr(AArch64::UZP1_ZZZ_H, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8489 }
8490 return 0;
8491}
8492
8493unsigned fastEmit_ISD_CONCAT_VECTORS_MVT_nxv2f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8494 if (RetVT.SimpleTy != MVT::nxv4f32)
8495 return 0;
8496 if ((Subtarget->hasSVE())) {
8497 return fastEmitInst_rr(AArch64::UZP1_ZZZ_S, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8498 }
8499 return 0;
8500}
8501
8502unsigned fastEmit_ISD_CONCAT_VECTORS_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8503 switch (VT.SimpleTy) {
8504 case MVT::nxv2i1: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv2i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8505 case MVT::nxv4i1: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv4i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8506 case MVT::nxv8i1: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv8i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8507 case MVT::nxv2f16: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv2f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8508 case MVT::nxv4f16: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv4f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8509 case MVT::nxv2bf16: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv2bf16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8510 case MVT::nxv4bf16: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv4bf16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8511 case MVT::nxv2f32: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv2f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8512 default: return 0;
8513 }
8514}
8515
8516// FastEmit functions for ISD::FADD.
8517
8518unsigned fastEmit_ISD_FADD_MVT_f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8519 if (RetVT.SimpleTy != MVT::f16)
8520 return 0;
8521 if ((Subtarget->hasFullFP16())) {
8522 return fastEmitInst_rr(AArch64::FADDHrr, &AArch64::FPR16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8523 }
8524 return 0;
8525}
8526
8527unsigned fastEmit_ISD_FADD_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8528 if (RetVT.SimpleTy != MVT::f32)
8529 return 0;
8530 if ((Subtarget->hasFPARMv8())) {
8531 return fastEmitInst_rr(AArch64::FADDSrr, &AArch64::FPR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8532 }
8533 return 0;
8534}
8535
8536unsigned fastEmit_ISD_FADD_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8537 if (RetVT.SimpleTy != MVT::f64)
8538 return 0;
8539 if ((Subtarget->hasFPARMv8())) {
8540 return fastEmitInst_rr(AArch64::FADDDrr, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8541 }
8542 return 0;
8543}
8544
8545unsigned fastEmit_ISD_FADD_MVT_v4f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8546 if (RetVT.SimpleTy != MVT::v4f16)
8547 return 0;
8548 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
8549 return fastEmitInst_rr(AArch64::FADDv4f16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8550 }
8551 return 0;
8552}
8553
8554unsigned fastEmit_ISD_FADD_MVT_v8f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8555 if (RetVT.SimpleTy != MVT::v8f16)
8556 return 0;
8557 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
8558 return fastEmitInst_rr(AArch64::FADDv8f16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8559 }
8560 return 0;
8561}
8562
8563unsigned fastEmit_ISD_FADD_MVT_v2f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8564 if (RetVT.SimpleTy != MVT::v2f32)
8565 return 0;
8566 if ((Subtarget->hasNEON())) {
8567 return fastEmitInst_rr(AArch64::FADDv2f32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8568 }
8569 return 0;
8570}
8571
8572unsigned fastEmit_ISD_FADD_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8573 if (RetVT.SimpleTy != MVT::v4f32)
8574 return 0;
8575 if ((Subtarget->hasNEON())) {
8576 return fastEmitInst_rr(AArch64::FADDv4f32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8577 }
8578 return 0;
8579}
8580
8581unsigned fastEmit_ISD_FADD_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8582 if (RetVT.SimpleTy != MVT::v2f64)
8583 return 0;
8584 if ((Subtarget->hasNEON())) {
8585 return fastEmitInst_rr(AArch64::FADDv2f64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8586 }
8587 return 0;
8588}
8589
8590unsigned fastEmit_ISD_FADD_MVT_nxv8f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8591 if (RetVT.SimpleTy != MVT::nxv8f16)
8592 return 0;
8593 if ((Subtarget->hasSVE())) {
8594 return fastEmitInst_rr(AArch64::FADD_ZZZ_H, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8595 }
8596 return 0;
8597}
8598
8599unsigned fastEmit_ISD_FADD_MVT_nxv4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8600 if (RetVT.SimpleTy != MVT::nxv4f32)
8601 return 0;
8602 if ((Subtarget->hasSVE())) {
8603 return fastEmitInst_rr(AArch64::FADD_ZZZ_S, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8604 }
8605 return 0;
8606}
8607
8608unsigned fastEmit_ISD_FADD_MVT_nxv2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8609 if (RetVT.SimpleTy != MVT::nxv2f64)
8610 return 0;
8611 if ((Subtarget->hasSVE())) {
8612 return fastEmitInst_rr(AArch64::FADD_ZZZ_D, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8613 }
8614 return 0;
8615}
8616
8617unsigned fastEmit_ISD_FADD_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8618 switch (VT.SimpleTy) {
8619 case MVT::f16: return fastEmit_ISD_FADD_MVT_f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8620 case MVT::f32: return fastEmit_ISD_FADD_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8621 case MVT::f64: return fastEmit_ISD_FADD_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8622 case MVT::v4f16: return fastEmit_ISD_FADD_MVT_v4f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8623 case MVT::v8f16: return fastEmit_ISD_FADD_MVT_v8f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8624 case MVT::v2f32: return fastEmit_ISD_FADD_MVT_v2f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8625 case MVT::v4f32: return fastEmit_ISD_FADD_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8626 case MVT::v2f64: return fastEmit_ISD_FADD_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8627 case MVT::nxv8f16: return fastEmit_ISD_FADD_MVT_nxv8f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8628 case MVT::nxv4f32: return fastEmit_ISD_FADD_MVT_nxv4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8629 case MVT::nxv2f64: return fastEmit_ISD_FADD_MVT_nxv2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8630 default: return 0;
8631 }
8632}
8633
8634// FastEmit functions for ISD::FDIV.
8635
8636unsigned fastEmit_ISD_FDIV_MVT_f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8637 if (RetVT.SimpleTy != MVT::f16)
8638 return 0;
8639 if ((Subtarget->hasFullFP16())) {
8640 return fastEmitInst_rr(AArch64::FDIVHrr, &AArch64::FPR16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8641 }
8642 return 0;
8643}
8644
8645unsigned fastEmit_ISD_FDIV_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8646 if (RetVT.SimpleTy != MVT::f32)
8647 return 0;
8648 if ((Subtarget->hasFPARMv8())) {
8649 return fastEmitInst_rr(AArch64::FDIVSrr, &AArch64::FPR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8650 }
8651 return 0;
8652}
8653
8654unsigned fastEmit_ISD_FDIV_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8655 if (RetVT.SimpleTy != MVT::f64)
8656 return 0;
8657 if ((Subtarget->hasFPARMv8())) {
8658 return fastEmitInst_rr(AArch64::FDIVDrr, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8659 }
8660 return 0;
8661}
8662
8663unsigned fastEmit_ISD_FDIV_MVT_v4f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8664 if (RetVT.SimpleTy != MVT::v4f16)
8665 return 0;
8666 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
8667 return fastEmitInst_rr(AArch64::FDIVv4f16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8668 }
8669 return 0;
8670}
8671
8672unsigned fastEmit_ISD_FDIV_MVT_v8f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8673 if (RetVT.SimpleTy != MVT::v8f16)
8674 return 0;
8675 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
8676 return fastEmitInst_rr(AArch64::FDIVv8f16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8677 }
8678 return 0;
8679}
8680
8681unsigned fastEmit_ISD_FDIV_MVT_v2f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8682 if (RetVT.SimpleTy != MVT::v2f32)
8683 return 0;
8684 if ((Subtarget->hasNEON())) {
8685 return fastEmitInst_rr(AArch64::FDIVv2f32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8686 }
8687 return 0;
8688}
8689
8690unsigned fastEmit_ISD_FDIV_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8691 if (RetVT.SimpleTy != MVT::v4f32)
8692 return 0;
8693 if ((Subtarget->hasNEON())) {
8694 return fastEmitInst_rr(AArch64::FDIVv4f32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8695 }
8696 return 0;
8697}
8698
8699unsigned fastEmit_ISD_FDIV_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8700 if (RetVT.SimpleTy != MVT::v2f64)
8701 return 0;
8702 if ((Subtarget->hasNEON())) {
8703 return fastEmitInst_rr(AArch64::FDIVv2f64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8704 }
8705 return 0;
8706}
8707
8708unsigned fastEmit_ISD_FDIV_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8709 switch (VT.SimpleTy) {
8710 case MVT::f16: return fastEmit_ISD_FDIV_MVT_f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8711 case MVT::f32: return fastEmit_ISD_FDIV_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8712 case MVT::f64: return fastEmit_ISD_FDIV_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8713 case MVT::v4f16: return fastEmit_ISD_FDIV_MVT_v4f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8714 case MVT::v8f16: return fastEmit_ISD_FDIV_MVT_v8f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8715 case MVT::v2f32: return fastEmit_ISD_FDIV_MVT_v2f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8716 case MVT::v4f32: return fastEmit_ISD_FDIV_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8717 case MVT::v2f64: return fastEmit_ISD_FDIV_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8718 default: return 0;
8719 }
8720}
8721
8722// FastEmit functions for ISD::FMAXIMUM.
8723
8724unsigned fastEmit_ISD_FMAXIMUM_MVT_f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8725 if (RetVT.SimpleTy != MVT::f16)
8726 return 0;
8727 if ((Subtarget->hasFullFP16())) {
8728 return fastEmitInst_rr(AArch64::FMAXHrr, &AArch64::FPR16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8729 }
8730 return 0;
8731}
8732
8733unsigned fastEmit_ISD_FMAXIMUM_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8734 if (RetVT.SimpleTy != MVT::f32)
8735 return 0;
8736 if ((Subtarget->hasFPARMv8())) {
8737 return fastEmitInst_rr(AArch64::FMAXSrr, &AArch64::FPR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8738 }
8739 return 0;
8740}
8741
8742unsigned fastEmit_ISD_FMAXIMUM_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8743 if (RetVT.SimpleTy != MVT::f64)
8744 return 0;
8745 if ((Subtarget->hasFPARMv8())) {
8746 return fastEmitInst_rr(AArch64::FMAXDrr, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8747 }
8748 return 0;
8749}
8750
8751unsigned fastEmit_ISD_FMAXIMUM_MVT_v4f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8752 if (RetVT.SimpleTy != MVT::v4f16)
8753 return 0;
8754 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
8755 return fastEmitInst_rr(AArch64::FMAXv4f16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8756 }
8757 return 0;
8758}
8759
8760unsigned fastEmit_ISD_FMAXIMUM_MVT_v8f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8761 if (RetVT.SimpleTy != MVT::v8f16)
8762 return 0;
8763 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
8764 return fastEmitInst_rr(AArch64::FMAXv8f16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8765 }
8766 return 0;
8767}
8768
8769unsigned fastEmit_ISD_FMAXIMUM_MVT_v2f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8770 if (RetVT.SimpleTy != MVT::v2f32)
8771 return 0;
8772 if ((Subtarget->hasNEON())) {
8773 return fastEmitInst_rr(AArch64::FMAXv2f32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8774 }
8775 return 0;
8776}
8777
8778unsigned fastEmit_ISD_FMAXIMUM_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8779 if (RetVT.SimpleTy != MVT::v4f32)
8780 return 0;
8781 if ((Subtarget->hasNEON())) {
8782 return fastEmitInst_rr(AArch64::FMAXv4f32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8783 }
8784 return 0;
8785}
8786
8787unsigned fastEmit_ISD_FMAXIMUM_MVT_v1f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8788 if (RetVT.SimpleTy != MVT::v1f64)
8789 return 0;
8790 return fastEmitInst_rr(AArch64::FMAXDrr, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8791}
8792
8793unsigned fastEmit_ISD_FMAXIMUM_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8794 if (RetVT.SimpleTy != MVT::v2f64)
8795 return 0;
8796 if ((Subtarget->hasNEON())) {
8797 return fastEmitInst_rr(AArch64::FMAXv2f64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8798 }
8799 return 0;
8800}
8801
8802unsigned fastEmit_ISD_FMAXIMUM_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8803 switch (VT.SimpleTy) {
8804 case MVT::f16: return fastEmit_ISD_FMAXIMUM_MVT_f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8805 case MVT::f32: return fastEmit_ISD_FMAXIMUM_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8806 case MVT::f64: return fastEmit_ISD_FMAXIMUM_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8807 case MVT::v4f16: return fastEmit_ISD_FMAXIMUM_MVT_v4f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8808 case MVT::v8f16: return fastEmit_ISD_FMAXIMUM_MVT_v8f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8809 case MVT::v2f32: return fastEmit_ISD_FMAXIMUM_MVT_v2f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8810 case MVT::v4f32: return fastEmit_ISD_FMAXIMUM_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8811 case MVT::v1f64: return fastEmit_ISD_FMAXIMUM_MVT_v1f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8812 case MVT::v2f64: return fastEmit_ISD_FMAXIMUM_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8813 default: return 0;
8814 }
8815}
8816
8817// FastEmit functions for ISD::FMAXNUM.
8818
8819unsigned fastEmit_ISD_FMAXNUM_MVT_f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8820 if (RetVT.SimpleTy != MVT::f16)
8821 return 0;
8822 if ((Subtarget->hasFullFP16())) {
8823 return fastEmitInst_rr(AArch64::FMAXNMHrr, &AArch64::FPR16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8824 }
8825 return 0;
8826}
8827
8828unsigned fastEmit_ISD_FMAXNUM_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8829 if (RetVT.SimpleTy != MVT::f32)
8830 return 0;
8831 if ((Subtarget->hasFPARMv8())) {
8832 return fastEmitInst_rr(AArch64::FMAXNMSrr, &AArch64::FPR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8833 }
8834 return 0;
8835}
8836
8837unsigned fastEmit_ISD_FMAXNUM_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8838 if (RetVT.SimpleTy != MVT::f64)
8839 return 0;
8840 if ((Subtarget->hasFPARMv8())) {
8841 return fastEmitInst_rr(AArch64::FMAXNMDrr, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8842 }
8843 return 0;
8844}
8845
8846unsigned fastEmit_ISD_FMAXNUM_MVT_v4f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8847 if (RetVT.SimpleTy != MVT::v4f16)
8848 return 0;
8849 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
8850 return fastEmitInst_rr(AArch64::FMAXNMv4f16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8851 }
8852 return 0;
8853}
8854
8855unsigned fastEmit_ISD_FMAXNUM_MVT_v8f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8856 if (RetVT.SimpleTy != MVT::v8f16)
8857 return 0;
8858 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
8859 return fastEmitInst_rr(AArch64::FMAXNMv8f16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8860 }
8861 return 0;
8862}
8863
8864unsigned fastEmit_ISD_FMAXNUM_MVT_v2f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8865 if (RetVT.SimpleTy != MVT::v2f32)
8866 return 0;
8867 if ((Subtarget->hasNEON())) {
8868 return fastEmitInst_rr(AArch64::FMAXNMv2f32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8869 }
8870 return 0;
8871}
8872
8873unsigned fastEmit_ISD_FMAXNUM_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8874 if (RetVT.SimpleTy != MVT::v4f32)
8875 return 0;
8876 if ((Subtarget->hasNEON())) {
8877 return fastEmitInst_rr(AArch64::FMAXNMv4f32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8878 }
8879 return 0;
8880}
8881
8882unsigned fastEmit_ISD_FMAXNUM_MVT_v1f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8883 if (RetVT.SimpleTy != MVT::v1f64)
8884 return 0;
8885 return fastEmitInst_rr(AArch64::FMAXNMDrr, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8886}
8887
8888unsigned fastEmit_ISD_FMAXNUM_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8889 if (RetVT.SimpleTy != MVT::v2f64)
8890 return 0;
8891 if ((Subtarget->hasNEON())) {
8892 return fastEmitInst_rr(AArch64::FMAXNMv2f64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8893 }
8894 return 0;
8895}
8896
8897unsigned fastEmit_ISD_FMAXNUM_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8898 switch (VT.SimpleTy) {
8899 case MVT::f16: return fastEmit_ISD_FMAXNUM_MVT_f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8900 case MVT::f32: return fastEmit_ISD_FMAXNUM_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8901 case MVT::f64: return fastEmit_ISD_FMAXNUM_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8902 case MVT::v4f16: return fastEmit_ISD_FMAXNUM_MVT_v4f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8903 case MVT::v8f16: return fastEmit_ISD_FMAXNUM_MVT_v8f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8904 case MVT::v2f32: return fastEmit_ISD_FMAXNUM_MVT_v2f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8905 case MVT::v4f32: return fastEmit_ISD_FMAXNUM_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8906 case MVT::v1f64: return fastEmit_ISD_FMAXNUM_MVT_v1f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8907 case MVT::v2f64: return fastEmit_ISD_FMAXNUM_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8908 default: return 0;
8909 }
8910}
8911
8912// FastEmit functions for ISD::FMINIMUM.
8913
8914unsigned fastEmit_ISD_FMINIMUM_MVT_f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8915 if (RetVT.SimpleTy != MVT::f16)
8916 return 0;
8917 if ((Subtarget->hasFullFP16())) {
8918 return fastEmitInst_rr(AArch64::FMINHrr, &AArch64::FPR16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8919 }
8920 return 0;
8921}
8922
8923unsigned fastEmit_ISD_FMINIMUM_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8924 if (RetVT.SimpleTy != MVT::f32)
8925 return 0;
8926 if ((Subtarget->hasFPARMv8())) {
8927 return fastEmitInst_rr(AArch64::FMINSrr, &AArch64::FPR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8928 }
8929 return 0;
8930}
8931
8932unsigned fastEmit_ISD_FMINIMUM_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8933 if (RetVT.SimpleTy != MVT::f64)
8934 return 0;
8935 if ((Subtarget->hasFPARMv8())) {
8936 return fastEmitInst_rr(AArch64::FMINDrr, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8937 }
8938 return 0;
8939}
8940
8941unsigned fastEmit_ISD_FMINIMUM_MVT_v4f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8942 if (RetVT.SimpleTy != MVT::v4f16)
8943 return 0;
8944 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
8945 return fastEmitInst_rr(AArch64::FMINv4f16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8946 }
8947 return 0;
8948}
8949
8950unsigned fastEmit_ISD_FMINIMUM_MVT_v8f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8951 if (RetVT.SimpleTy != MVT::v8f16)
8952 return 0;
8953 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
8954 return fastEmitInst_rr(AArch64::FMINv8f16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8955 }
8956 return 0;
8957}
8958
8959unsigned fastEmit_ISD_FMINIMUM_MVT_v2f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8960 if (RetVT.SimpleTy != MVT::v2f32)
8961 return 0;
8962 if ((Subtarget->hasNEON())) {
8963 return fastEmitInst_rr(AArch64::FMINv2f32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8964 }
8965 return 0;
8966}
8967
8968unsigned fastEmit_ISD_FMINIMUM_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8969 if (RetVT.SimpleTy != MVT::v4f32)
8970 return 0;
8971 if ((Subtarget->hasNEON())) {
8972 return fastEmitInst_rr(AArch64::FMINv4f32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8973 }
8974 return 0;
8975}
8976
8977unsigned fastEmit_ISD_FMINIMUM_MVT_v1f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8978 if (RetVT.SimpleTy != MVT::v1f64)
8979 return 0;
8980 return fastEmitInst_rr(AArch64::FMINDrr, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8981}
8982
8983unsigned fastEmit_ISD_FMINIMUM_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8984 if (RetVT.SimpleTy != MVT::v2f64)
8985 return 0;
8986 if ((Subtarget->hasNEON())) {
8987 return fastEmitInst_rr(AArch64::FMINv2f64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8988 }
8989 return 0;
8990}
8991
8992unsigned fastEmit_ISD_FMINIMUM_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8993 switch (VT.SimpleTy) {
8994 case MVT::f16: return fastEmit_ISD_FMINIMUM_MVT_f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8995 case MVT::f32: return fastEmit_ISD_FMINIMUM_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8996 case MVT::f64: return fastEmit_ISD_FMINIMUM_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8997 case MVT::v4f16: return fastEmit_ISD_FMINIMUM_MVT_v4f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8998 case MVT::v8f16: return fastEmit_ISD_FMINIMUM_MVT_v8f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8999 case MVT::v2f32: return fastEmit_ISD_FMINIMUM_MVT_v2f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9000 case MVT::v4f32: return fastEmit_ISD_FMINIMUM_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9001 case MVT::v1f64: return fastEmit_ISD_FMINIMUM_MVT_v1f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9002 case MVT::v2f64: return fastEmit_ISD_FMINIMUM_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9003 default: return 0;
9004 }
9005}
9006
9007// FastEmit functions for ISD::FMINNUM.
9008
9009unsigned fastEmit_ISD_FMINNUM_MVT_f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9010 if (RetVT.SimpleTy != MVT::f16)
9011 return 0;
9012 if ((Subtarget->hasFullFP16())) {
9013 return fastEmitInst_rr(AArch64::FMINNMHrr, &AArch64::FPR16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9014 }
9015 return 0;
9016}
9017
9018unsigned fastEmit_ISD_FMINNUM_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9019 if (RetVT.SimpleTy != MVT::f32)
9020 return 0;
9021 if ((Subtarget->hasFPARMv8())) {
9022 return fastEmitInst_rr(AArch64::FMINNMSrr, &AArch64::FPR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9023 }
9024 return 0;
9025}
9026
9027unsigned fastEmit_ISD_FMINNUM_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9028 if (RetVT.SimpleTy != MVT::f64)
9029 return 0;
9030 if ((Subtarget->hasFPARMv8())) {
9031 return fastEmitInst_rr(AArch64::FMINNMDrr, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9032 }
9033 return 0;
9034}
9035
9036unsigned fastEmit_ISD_FMINNUM_MVT_v4f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9037 if (RetVT.SimpleTy != MVT::v4f16)
9038 return 0;
9039 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
9040 return fastEmitInst_rr(AArch64::FMINNMv4f16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9041 }
9042 return 0;
9043}
9044
9045unsigned fastEmit_ISD_FMINNUM_MVT_v8f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9046 if (RetVT.SimpleTy != MVT::v8f16)
9047 return 0;
9048 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
9049 return fastEmitInst_rr(AArch64::FMINNMv8f16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9050 }
9051 return 0;
9052}
9053
9054unsigned fastEmit_ISD_FMINNUM_MVT_v2f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9055 if (RetVT.SimpleTy != MVT::v2f32)
9056 return 0;
9057 if ((Subtarget->hasNEON())) {
9058 return fastEmitInst_rr(AArch64::FMINNMv2f32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9059 }
9060 return 0;
9061}
9062
9063unsigned fastEmit_ISD_FMINNUM_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9064 if (RetVT.SimpleTy != MVT::v4f32)
9065 return 0;
9066 if ((Subtarget->hasNEON())) {
9067 return fastEmitInst_rr(AArch64::FMINNMv4f32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9068 }
9069 return 0;
9070}
9071
9072unsigned fastEmit_ISD_FMINNUM_MVT_v1f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9073 if (RetVT.SimpleTy != MVT::v1f64)
9074 return 0;
9075 return fastEmitInst_rr(AArch64::FMINNMDrr, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9076}
9077
9078unsigned fastEmit_ISD_FMINNUM_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9079 if (RetVT.SimpleTy != MVT::v2f64)
9080 return 0;
9081 if ((Subtarget->hasNEON())) {
9082 return fastEmitInst_rr(AArch64::FMINNMv2f64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9083 }
9084 return 0;
9085}
9086
9087unsigned fastEmit_ISD_FMINNUM_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9088 switch (VT.SimpleTy) {
9089 case MVT::f16: return fastEmit_ISD_FMINNUM_MVT_f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9090 case MVT::f32: return fastEmit_ISD_FMINNUM_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9091 case MVT::f64: return fastEmit_ISD_FMINNUM_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9092 case MVT::v4f16: return fastEmit_ISD_FMINNUM_MVT_v4f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9093 case MVT::v8f16: return fastEmit_ISD_FMINNUM_MVT_v8f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9094 case MVT::v2f32: return fastEmit_ISD_FMINNUM_MVT_v2f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9095 case MVT::v4f32: return fastEmit_ISD_FMINNUM_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9096 case MVT::v1f64: return fastEmit_ISD_FMINNUM_MVT_v1f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9097 case MVT::v2f64: return fastEmit_ISD_FMINNUM_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9098 default: return 0;
9099 }
9100}
9101
9102// FastEmit functions for ISD::FMUL.
9103
9104unsigned fastEmit_ISD_FMUL_MVT_f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9105 if (RetVT.SimpleTy != MVT::f16)
9106 return 0;
9107 if ((Subtarget->hasFullFP16())) {
9108 return fastEmitInst_rr(AArch64::FMULHrr, &AArch64::FPR16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9109 }
9110 return 0;
9111}
9112
9113unsigned fastEmit_ISD_FMUL_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9114 if (RetVT.SimpleTy != MVT::f32)
9115 return 0;
9116 if ((Subtarget->hasFPARMv8())) {
9117 return fastEmitInst_rr(AArch64::FMULSrr, &AArch64::FPR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9118 }
9119 return 0;
9120}
9121
9122unsigned fastEmit_ISD_FMUL_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9123 if (RetVT.SimpleTy != MVT::f64)
9124 return 0;
9125 if ((Subtarget->hasFPARMv8())) {
9126 return fastEmitInst_rr(AArch64::FMULDrr, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9127 }
9128 return 0;
9129}
9130
9131unsigned fastEmit_ISD_FMUL_MVT_v4f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9132 if (RetVT.SimpleTy != MVT::v4f16)
9133 return 0;
9134 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
9135 return fastEmitInst_rr(AArch64::FMULv4f16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9136 }
9137 return 0;
9138}
9139
9140unsigned fastEmit_ISD_FMUL_MVT_v8f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9141 if (RetVT.SimpleTy != MVT::v8f16)
9142 return 0;
9143 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
9144 return fastEmitInst_rr(AArch64::FMULv8f16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9145 }
9146 return 0;
9147}
9148
9149unsigned fastEmit_ISD_FMUL_MVT_v2f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9150 if (RetVT.SimpleTy != MVT::v2f32)
9151 return 0;
9152 if ((Subtarget->hasNEON())) {
9153 return fastEmitInst_rr(AArch64::FMULv2f32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9154 }
9155 return 0;
9156}
9157
9158unsigned fastEmit_ISD_FMUL_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9159 if (RetVT.SimpleTy != MVT::v4f32)
9160 return 0;
9161 if ((Subtarget->hasNEON())) {
9162 return fastEmitInst_rr(AArch64::FMULv4f32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9163 }
9164 return 0;
9165}
9166
9167unsigned fastEmit_ISD_FMUL_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9168 if (RetVT.SimpleTy != MVT::v2f64)
9169 return 0;
9170 if ((Subtarget->hasNEON())) {
9171 return fastEmitInst_rr(AArch64::FMULv2f64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9172 }
9173 return 0;
9174}
9175
9176unsigned fastEmit_ISD_FMUL_MVT_nxv8f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9177 if (RetVT.SimpleTy != MVT::nxv8f16)
9178 return 0;
9179 if ((Subtarget->hasSVE())) {
9180 return fastEmitInst_rr(AArch64::FMUL_ZZZ_H, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9181 }
9182 return 0;
9183}
9184
9185unsigned fastEmit_ISD_FMUL_MVT_nxv4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9186 if (RetVT.SimpleTy != MVT::nxv4f32)
9187 return 0;
9188 if ((Subtarget->hasSVE())) {
9189 return fastEmitInst_rr(AArch64::FMUL_ZZZ_S, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9190 }
9191 return 0;
9192}
9193
9194unsigned fastEmit_ISD_FMUL_MVT_nxv2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9195 if (RetVT.SimpleTy != MVT::nxv2f64)
9196 return 0;
9197 if ((Subtarget->hasSVE())) {
9198 return fastEmitInst_rr(AArch64::FMUL_ZZZ_D, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9199 }
9200 return 0;
9201}
9202
9203unsigned fastEmit_ISD_FMUL_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9204 switch (VT.SimpleTy) {
9205 case MVT::f16: return fastEmit_ISD_FMUL_MVT_f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9206 case MVT::f32: return fastEmit_ISD_FMUL_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9207 case MVT::f64: return fastEmit_ISD_FMUL_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9208 case MVT::v4f16: return fastEmit_ISD_FMUL_MVT_v4f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9209 case MVT::v8f16: return fastEmit_ISD_FMUL_MVT_v8f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9210 case MVT::v2f32: return fastEmit_ISD_FMUL_MVT_v2f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9211 case MVT::v4f32: return fastEmit_ISD_FMUL_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9212 case MVT::v2f64: return fastEmit_ISD_FMUL_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9213 case MVT::nxv8f16: return fastEmit_ISD_FMUL_MVT_nxv8f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9214 case MVT::nxv4f32: return fastEmit_ISD_FMUL_MVT_nxv4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9215 case MVT::nxv2f64: return fastEmit_ISD_FMUL_MVT_nxv2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9216 default: return 0;
9217 }
9218}
9219
9220// FastEmit functions for ISD::FSUB.
9221
9222unsigned fastEmit_ISD_FSUB_MVT_f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9223 if (RetVT.SimpleTy != MVT::f16)
9224 return 0;
9225 if ((Subtarget->hasFullFP16())) {
9226 return fastEmitInst_rr(AArch64::FSUBHrr, &AArch64::FPR16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9227 }
9228 return 0;
9229}
9230
9231unsigned fastEmit_ISD_FSUB_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9232 if (RetVT.SimpleTy != MVT::f32)
9233 return 0;
9234 if ((Subtarget->hasFPARMv8())) {
9235 return fastEmitInst_rr(AArch64::FSUBSrr, &AArch64::FPR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9236 }
9237 return 0;
9238}
9239
9240unsigned fastEmit_ISD_FSUB_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9241 if (RetVT.SimpleTy != MVT::f64)
9242 return 0;
9243 if ((Subtarget->hasFPARMv8())) {
9244 return fastEmitInst_rr(AArch64::FSUBDrr, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9245 }
9246 return 0;
9247}
9248
9249unsigned fastEmit_ISD_FSUB_MVT_v4f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9250 if (RetVT.SimpleTy != MVT::v4f16)
9251 return 0;
9252 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
9253 return fastEmitInst_rr(AArch64::FSUBv4f16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9254 }
9255 return 0;
9256}
9257
9258unsigned fastEmit_ISD_FSUB_MVT_v8f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9259 if (RetVT.SimpleTy != MVT::v8f16)
9260 return 0;
9261 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
9262 return fastEmitInst_rr(AArch64::FSUBv8f16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9263 }
9264 return 0;
9265}
9266
9267unsigned fastEmit_ISD_FSUB_MVT_v2f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9268 if (RetVT.SimpleTy != MVT::v2f32)
9269 return 0;
9270 if ((Subtarget->hasNEON())) {
9271 return fastEmitInst_rr(AArch64::FSUBv2f32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9272 }
9273 return 0;
9274}
9275
9276unsigned fastEmit_ISD_FSUB_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9277 if (RetVT.SimpleTy != MVT::v4f32)
9278 return 0;
9279 if ((Subtarget->hasNEON())) {
9280 return fastEmitInst_rr(AArch64::FSUBv4f32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9281 }
9282 return 0;
9283}
9284
9285unsigned fastEmit_ISD_FSUB_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9286 if (RetVT.SimpleTy != MVT::v2f64)
9287 return 0;
9288 if ((Subtarget->hasNEON())) {
9289 return fastEmitInst_rr(AArch64::FSUBv2f64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9290 }
9291 return 0;
9292}
9293
9294unsigned fastEmit_ISD_FSUB_MVT_nxv8f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9295 if (RetVT.SimpleTy != MVT::nxv8f16)
9296 return 0;
9297 if ((Subtarget->hasSVE())) {
9298 return fastEmitInst_rr(AArch64::FSUB_ZZZ_H, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9299 }
9300 return 0;
9301}
9302
9303unsigned fastEmit_ISD_FSUB_MVT_nxv4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9304 if (RetVT.SimpleTy != MVT::nxv4f32)
9305 return 0;
9306 if ((Subtarget->hasSVE())) {
9307 return fastEmitInst_rr(AArch64::FSUB_ZZZ_S, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9308 }
9309 return 0;
9310}
9311
9312unsigned fastEmit_ISD_FSUB_MVT_nxv2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9313 if (RetVT.SimpleTy != MVT::nxv2f64)
9314 return 0;
9315 if ((Subtarget->hasSVE())) {
9316 return fastEmitInst_rr(AArch64::FSUB_ZZZ_D, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9317 }
9318 return 0;
9319}
9320
9321unsigned fastEmit_ISD_FSUB_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9322 switch (VT.SimpleTy) {
9323 case MVT::f16: return fastEmit_ISD_FSUB_MVT_f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9324 case MVT::f32: return fastEmit_ISD_FSUB_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9325 case MVT::f64: return fastEmit_ISD_FSUB_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9326 case MVT::v4f16: return fastEmit_ISD_FSUB_MVT_v4f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9327 case MVT::v8f16: return fastEmit_ISD_FSUB_MVT_v8f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9328 case MVT::v2f32: return fastEmit_ISD_FSUB_MVT_v2f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9329 case MVT::v4f32: return fastEmit_ISD_FSUB_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9330 case MVT::v2f64: return fastEmit_ISD_FSUB_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9331 case MVT::nxv8f16: return fastEmit_ISD_FSUB_MVT_nxv8f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9332 case MVT::nxv4f32: return fastEmit_ISD_FSUB_MVT_nxv4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9333 case MVT::nxv2f64: return fastEmit_ISD_FSUB_MVT_nxv2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9334 default: return 0;
9335 }
9336}
9337
9338// FastEmit functions for ISD::MUL.
9339
9340unsigned fastEmit_ISD_MUL_MVT_v8i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9341 if (RetVT.SimpleTy != MVT::v8i8)
9342 return 0;
9343 if ((Subtarget->hasNEON())) {
9344 return fastEmitInst_rr(AArch64::MULv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9345 }
9346 return 0;
9347}
9348
9349unsigned fastEmit_ISD_MUL_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9350 if (RetVT.SimpleTy != MVT::v16i8)
9351 return 0;
9352 if ((Subtarget->hasNEON())) {
9353 return fastEmitInst_rr(AArch64::MULv16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9354 }
9355 return 0;
9356}
9357
9358unsigned fastEmit_ISD_MUL_MVT_v4i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9359 if (RetVT.SimpleTy != MVT::v4i16)
9360 return 0;
9361 if ((Subtarget->hasNEON())) {
9362 return fastEmitInst_rr(AArch64::MULv4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9363 }
9364 return 0;
9365}
9366
9367unsigned fastEmit_ISD_MUL_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9368 if (RetVT.SimpleTy != MVT::v8i16)
9369 return 0;
9370 if ((Subtarget->hasNEON())) {
9371 return fastEmitInst_rr(AArch64::MULv8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9372 }
9373 return 0;
9374}
9375
9376unsigned fastEmit_ISD_MUL_MVT_v2i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9377 if (RetVT.SimpleTy != MVT::v2i32)
9378 return 0;
9379 if ((Subtarget->hasNEON())) {
9380 return fastEmitInst_rr(AArch64::MULv2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9381 }
9382 return 0;
9383}
9384
9385unsigned fastEmit_ISD_MUL_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9386 if (RetVT.SimpleTy != MVT::v4i32)
9387 return 0;
9388 if ((Subtarget->hasNEON())) {
9389 return fastEmitInst_rr(AArch64::MULv4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9390 }
9391 return 0;
9392}
9393
9394unsigned fastEmit_ISD_MUL_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9395 switch (VT.SimpleTy) {
9396 case MVT::v8i8: return fastEmit_ISD_MUL_MVT_v8i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9397 case MVT::v16i8: return fastEmit_ISD_MUL_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9398 case MVT::v4i16: return fastEmit_ISD_MUL_MVT_v4i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9399 case MVT::v8i16: return fastEmit_ISD_MUL_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9400 case MVT::v2i32: return fastEmit_ISD_MUL_MVT_v2i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9401 case MVT::v4i32: return fastEmit_ISD_MUL_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9402 default: return 0;
9403 }
9404}
9405
9406// FastEmit functions for ISD::MULHS.
9407
9408unsigned fastEmit_ISD_MULHS_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9409 if (RetVT.SimpleTy != MVT::i64)
9410 return 0;
9411 return fastEmitInst_rr(AArch64::SMULHrr, &AArch64::GPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9412}
9413
9414unsigned fastEmit_ISD_MULHS_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9415 switch (VT.SimpleTy) {
9416 case MVT::i64: return fastEmit_ISD_MULHS_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9417 default: return 0;
9418 }
9419}
9420
9421// FastEmit functions for ISD::MULHU.
9422
9423unsigned fastEmit_ISD_MULHU_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9424 if (RetVT.SimpleTy != MVT::i64)
9425 return 0;
9426 return fastEmitInst_rr(AArch64::UMULHrr, &AArch64::GPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9427}
9428
9429unsigned fastEmit_ISD_MULHU_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9430 switch (VT.SimpleTy) {
9431 case MVT::i64: return fastEmit_ISD_MULHU_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9432 default: return 0;
9433 }
9434}
9435
9436// FastEmit functions for ISD::OR.
9437
9438unsigned fastEmit_ISD_OR_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9439 if (RetVT.SimpleTy != MVT::i32)
9440 return 0;
9441 return fastEmitInst_rr(AArch64::ORRWrr, &AArch64::GPR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9442}
9443
9444unsigned fastEmit_ISD_OR_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9445 if (RetVT.SimpleTy != MVT::i64)
9446 return 0;
9447 return fastEmitInst_rr(AArch64::ORRXrr, &AArch64::GPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9448}
9449
9450unsigned fastEmit_ISD_OR_MVT_v8i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9451 if (RetVT.SimpleTy != MVT::v8i8)
9452 return 0;
9453 if ((Subtarget->hasNEON())) {
9454 return fastEmitInst_rr(AArch64::ORRv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9455 }
9456 return 0;
9457}
9458
9459unsigned fastEmit_ISD_OR_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9460 if (RetVT.SimpleTy != MVT::v16i8)
9461 return 0;
9462 if ((Subtarget->hasNEON())) {
9463 return fastEmitInst_rr(AArch64::ORRv16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9464 }
9465 return 0;
9466}
9467
9468unsigned fastEmit_ISD_OR_MVT_v4i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9469 if (RetVT.SimpleTy != MVT::v4i16)
9470 return 0;
9471 if ((Subtarget->hasNEON())) {
9472 return fastEmitInst_rr(AArch64::ORRv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9473 }
9474 return 0;
9475}
9476
9477unsigned fastEmit_ISD_OR_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9478 if (RetVT.SimpleTy != MVT::v8i16)
9479 return 0;
9480 if ((Subtarget->hasNEON())) {
9481 return fastEmitInst_rr(AArch64::ORRv16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9482 }
9483 return 0;
9484}
9485
9486unsigned fastEmit_ISD_OR_MVT_v2i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9487 if (RetVT.SimpleTy != MVT::v2i32)
9488 return 0;
9489 if ((Subtarget->hasNEON())) {
9490 return fastEmitInst_rr(AArch64::ORRv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9491 }
9492 return 0;
9493}
9494
9495unsigned fastEmit_ISD_OR_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9496 if (RetVT.SimpleTy != MVT::v4i32)
9497 return 0;
9498 if ((Subtarget->hasNEON())) {
9499 return fastEmitInst_rr(AArch64::ORRv16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9500 }
9501 return 0;
9502}
9503
9504unsigned fastEmit_ISD_OR_MVT_v1i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9505 if (RetVT.SimpleTy != MVT::v1i64)
9506 return 0;
9507 if ((Subtarget->hasNEON())) {
9508 return fastEmitInst_rr(AArch64::ORRv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9509 }
9510 return 0;
9511}
9512
9513unsigned fastEmit_ISD_OR_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9514 if (RetVT.SimpleTy != MVT::v2i64)
9515 return 0;
9516 if ((Subtarget->hasNEON())) {
9517 return fastEmitInst_rr(AArch64::ORRv16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9518 }
9519 return 0;
9520}
9521
9522unsigned fastEmit_ISD_OR_MVT_nxv16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9523 if (RetVT.SimpleTy != MVT::nxv16i8)
9524 return 0;
9525 if ((Subtarget->hasSVE())) {
9526 return fastEmitInst_rr(AArch64::ORR_ZZZ, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9527 }
9528 return 0;
9529}
9530
9531unsigned fastEmit_ISD_OR_MVT_nxv8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9532 if (RetVT.SimpleTy != MVT::nxv8i16)
9533 return 0;
9534 if ((Subtarget->hasSVE())) {
9535 return fastEmitInst_rr(AArch64::ORR_ZZZ, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9536 }
9537 return 0;
9538}
9539
9540unsigned fastEmit_ISD_OR_MVT_nxv4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9541 if (RetVT.SimpleTy != MVT::nxv4i32)
9542 return 0;
9543 if ((Subtarget->hasSVE())) {
9544 return fastEmitInst_rr(AArch64::ORR_ZZZ, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9545 }
9546 return 0;
9547}
9548
9549unsigned fastEmit_ISD_OR_MVT_nxv2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9550 if (RetVT.SimpleTy != MVT::nxv2i64)
9551 return 0;
9552 if ((Subtarget->hasSVE())) {
9553 return fastEmitInst_rr(AArch64::ORR_ZZZ, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9554 }
9555 return 0;
9556}
9557
9558unsigned fastEmit_ISD_OR_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9559 switch (VT.SimpleTy) {
9560 case MVT::i32: return fastEmit_ISD_OR_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9561 case MVT::i64: return fastEmit_ISD_OR_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9562 case MVT::v8i8: return fastEmit_ISD_OR_MVT_v8i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9563 case MVT::v16i8: return fastEmit_ISD_OR_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9564 case MVT::v4i16: return fastEmit_ISD_OR_MVT_v4i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9565 case MVT::v8i16: return fastEmit_ISD_OR_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9566 case MVT::v2i32: return fastEmit_ISD_OR_MVT_v2i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9567 case MVT::v4i32: return fastEmit_ISD_OR_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9568 case MVT::v1i64: return fastEmit_ISD_OR_MVT_v1i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9569 case MVT::v2i64: return fastEmit_ISD_OR_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9570 case MVT::nxv16i8: return fastEmit_ISD_OR_MVT_nxv16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9571 case MVT::nxv8i16: return fastEmit_ISD_OR_MVT_nxv8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9572 case MVT::nxv4i32: return fastEmit_ISD_OR_MVT_nxv4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9573 case MVT::nxv2i64: return fastEmit_ISD_OR_MVT_nxv2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9574 default: return 0;
9575 }
9576}
9577
9578// FastEmit functions for ISD::ROTR.
9579
9580unsigned fastEmit_ISD_ROTR_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9581 if (RetVT.SimpleTy != MVT::i64)
9582 return 0;
9583 return fastEmitInst_rr(AArch64::RORVXr, &AArch64::GPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9584}
9585
9586unsigned fastEmit_ISD_ROTR_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9587 switch (VT.SimpleTy) {
9588 case MVT::i64: return fastEmit_ISD_ROTR_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9589 default: return 0;
9590 }
9591}
9592
9593// FastEmit functions for ISD::SADDSAT.
9594
9595unsigned fastEmit_ISD_SADDSAT_MVT_v8i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9596 if (RetVT.SimpleTy != MVT::v8i8)
9597 return 0;
9598 if ((Subtarget->hasNEON())) {
9599 return fastEmitInst_rr(AArch64::SQADDv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9600 }
9601 return 0;
9602}
9603
9604unsigned fastEmit_ISD_SADDSAT_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9605 if (RetVT.SimpleTy != MVT::v16i8)
9606 return 0;
9607 if ((Subtarget->hasNEON())) {
9608 return fastEmitInst_rr(AArch64::SQADDv16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9609 }
9610 return 0;
9611}
9612
9613unsigned fastEmit_ISD_SADDSAT_MVT_v4i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9614 if (RetVT.SimpleTy != MVT::v4i16)
9615 return 0;
9616 if ((Subtarget->hasNEON())) {
9617 return fastEmitInst_rr(AArch64::SQADDv4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9618 }
9619 return 0;
9620}
9621
9622unsigned fastEmit_ISD_SADDSAT_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9623 if (RetVT.SimpleTy != MVT::v8i16)
9624 return 0;
9625 if ((Subtarget->hasNEON())) {
9626 return fastEmitInst_rr(AArch64::SQADDv8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9627 }
9628 return 0;
9629}
9630
9631unsigned fastEmit_ISD_SADDSAT_MVT_v2i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9632 if (RetVT.SimpleTy != MVT::v2i32)
9633 return 0;
9634 if ((Subtarget->hasNEON())) {
9635 return fastEmitInst_rr(AArch64::SQADDv2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9636 }
9637 return 0;
9638}
9639
9640unsigned fastEmit_ISD_SADDSAT_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9641 if (RetVT.SimpleTy != MVT::v4i32)
9642 return 0;
9643 if ((Subtarget->hasNEON())) {
9644 return fastEmitInst_rr(AArch64::SQADDv4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9645 }
9646 return 0;
9647}
9648
9649unsigned fastEmit_ISD_SADDSAT_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9650 if (RetVT.SimpleTy != MVT::v2i64)
9651 return 0;
9652 if ((Subtarget->hasNEON())) {
9653 return fastEmitInst_rr(AArch64::SQADDv2i64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9654 }
9655 return 0;
9656}
9657
9658unsigned fastEmit_ISD_SADDSAT_MVT_nxv16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9659 if (RetVT.SimpleTy != MVT::nxv16i8)
9660 return 0;
9661 if ((Subtarget->hasSVE())) {
9662 return fastEmitInst_rr(AArch64::SQADD_ZZZ_B, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9663 }
9664 return 0;
9665}
9666
9667unsigned fastEmit_ISD_SADDSAT_MVT_nxv8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9668 if (RetVT.SimpleTy != MVT::nxv8i16)
9669 return 0;
9670 if ((Subtarget->hasSVE())) {
9671 return fastEmitInst_rr(AArch64::SQADD_ZZZ_H, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9672 }
9673 return 0;
9674}
9675
9676unsigned fastEmit_ISD_SADDSAT_MVT_nxv4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9677 if (RetVT.SimpleTy != MVT::nxv4i32)
9678 return 0;
9679 if ((Subtarget->hasSVE())) {
9680 return fastEmitInst_rr(AArch64::SQADD_ZZZ_S, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9681 }
9682 return 0;
9683}
9684
9685unsigned fastEmit_ISD_SADDSAT_MVT_nxv2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9686 if (RetVT.SimpleTy != MVT::nxv2i64)
9687 return 0;
9688 if ((Subtarget->hasSVE())) {
9689 return fastEmitInst_rr(AArch64::SQADD_ZZZ_D, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9690 }
9691 return 0;
9692}
9693
9694unsigned fastEmit_ISD_SADDSAT_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9695 switch (VT.SimpleTy) {
9696 case MVT::v8i8: return fastEmit_ISD_SADDSAT_MVT_v8i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9697 case MVT::v16i8: return fastEmit_ISD_SADDSAT_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9698 case MVT::v4i16: return fastEmit_ISD_SADDSAT_MVT_v4i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9699 case MVT::v8i16: return fastEmit_ISD_SADDSAT_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9700 case MVT::v2i32: return fastEmit_ISD_SADDSAT_MVT_v2i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9701 case MVT::v4i32: return fastEmit_ISD_SADDSAT_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9702 case MVT::v2i64: return fastEmit_ISD_SADDSAT_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9703 case MVT::nxv16i8: return fastEmit_ISD_SADDSAT_MVT_nxv16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9704 case MVT::nxv8i16: return fastEmit_ISD_SADDSAT_MVT_nxv8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9705 case MVT::nxv4i32: return fastEmit_ISD_SADDSAT_MVT_nxv4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9706 case MVT::nxv2i64: return fastEmit_ISD_SADDSAT_MVT_nxv2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9707 default: return 0;
9708 }
9709}
9710
9711// FastEmit functions for ISD::SDIV.
9712
9713unsigned fastEmit_ISD_SDIV_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9714 if (RetVT.SimpleTy != MVT::i32)
9715 return 0;
9716 return fastEmitInst_rr(AArch64::SDIVWr, &AArch64::GPR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9717}
9718
9719unsigned fastEmit_ISD_SDIV_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9720 if (RetVT.SimpleTy != MVT::i64)
9721 return 0;
9722 return fastEmitInst_rr(AArch64::SDIVXr, &AArch64::GPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9723}
9724
9725unsigned fastEmit_ISD_SDIV_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9726 switch (VT.SimpleTy) {
9727 case MVT::i32: return fastEmit_ISD_SDIV_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9728 case MVT::i64: return fastEmit_ISD_SDIV_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9729 default: return 0;
9730 }
9731}
9732
9733// FastEmit functions for ISD::SHL.
9734
9735unsigned fastEmit_ISD_SHL_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9736 if (RetVT.SimpleTy != MVT::i64)
9737 return 0;
9738 return fastEmitInst_rr(AArch64::LSLVXr, &AArch64::GPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9739}
9740
9741unsigned fastEmit_ISD_SHL_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9742 switch (VT.SimpleTy) {
9743 case MVT::i64: return fastEmit_ISD_SHL_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9744 default: return 0;
9745 }
9746}
9747
9748// FastEmit functions for ISD::SMAX.
9749
9750unsigned fastEmit_ISD_SMAX_MVT_v8i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9751 if (RetVT.SimpleTy != MVT::v8i8)
9752 return 0;
9753 if ((Subtarget->hasNEON())) {
9754 return fastEmitInst_rr(AArch64::SMAXv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9755 }
9756 return 0;
9757}
9758
9759unsigned fastEmit_ISD_SMAX_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9760 if (RetVT.SimpleTy != MVT::v16i8)
9761 return 0;
9762 if ((Subtarget->hasNEON())) {
9763 return fastEmitInst_rr(AArch64::SMAXv16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9764 }
9765 return 0;
9766}
9767
9768unsigned fastEmit_ISD_SMAX_MVT_v4i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9769 if (RetVT.SimpleTy != MVT::v4i16)
9770 return 0;
9771 if ((Subtarget->hasNEON())) {
9772 return fastEmitInst_rr(AArch64::SMAXv4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9773 }
9774 return 0;
9775}
9776
9777unsigned fastEmit_ISD_SMAX_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9778 if (RetVT.SimpleTy != MVT::v8i16)
9779 return 0;
9780 if ((Subtarget->hasNEON())) {
9781 return fastEmitInst_rr(AArch64::SMAXv8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9782 }
9783 return 0;
9784}
9785
9786unsigned fastEmit_ISD_SMAX_MVT_v2i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9787 if (RetVT.SimpleTy != MVT::v2i32)
9788 return 0;
9789 if ((Subtarget->hasNEON())) {
9790 return fastEmitInst_rr(AArch64::SMAXv2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9791 }
9792 return 0;
9793}
9794
9795unsigned fastEmit_ISD_SMAX_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9796 if (RetVT.SimpleTy != MVT::v4i32)
9797 return 0;
9798 if ((Subtarget->hasNEON())) {
9799 return fastEmitInst_rr(AArch64::SMAXv4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9800 }
9801 return 0;
9802}
9803
9804unsigned fastEmit_ISD_SMAX_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9805 switch (VT.SimpleTy) {
9806 case MVT::v8i8: return fastEmit_ISD_SMAX_MVT_v8i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9807 case MVT::v16i8: return fastEmit_ISD_SMAX_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9808 case MVT::v4i16: return fastEmit_ISD_SMAX_MVT_v4i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9809 case MVT::v8i16: return fastEmit_ISD_SMAX_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9810 case MVT::v2i32: return fastEmit_ISD_SMAX_MVT_v2i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9811 case MVT::v4i32: return fastEmit_ISD_SMAX_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9812 default: return 0;
9813 }
9814}
9815
9816// FastEmit functions for ISD::SMIN.
9817
9818unsigned fastEmit_ISD_SMIN_MVT_v8i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9819 if (RetVT.SimpleTy != MVT::v8i8)
9820 return 0;
9821 if ((Subtarget->hasNEON())) {
9822 return fastEmitInst_rr(AArch64::SMINv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9823 }
9824 return 0;
9825}
9826
9827unsigned fastEmit_ISD_SMIN_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9828 if (RetVT.SimpleTy != MVT::v16i8)
9829 return 0;
9830 if ((Subtarget->hasNEON())) {
9831 return fastEmitInst_rr(AArch64::SMINv16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9832 }
9833 return 0;
9834}
9835
9836unsigned fastEmit_ISD_SMIN_MVT_v4i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9837 if (RetVT.SimpleTy != MVT::v4i16)
9838 return 0;
9839 if ((Subtarget->hasNEON())) {
9840 return fastEmitInst_rr(AArch64::SMINv4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9841 }
9842 return 0;
9843}
9844
9845unsigned fastEmit_ISD_SMIN_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9846 if (RetVT.SimpleTy != MVT::v8i16)
9847 return 0;
9848 if ((Subtarget->hasNEON())) {
9849 return fastEmitInst_rr(AArch64::SMINv8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9850 }
9851 return 0;
9852}
9853
9854unsigned fastEmit_ISD_SMIN_MVT_v2i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9855 if (RetVT.SimpleTy != MVT::v2i32)
9856 return 0;
9857 if ((Subtarget->hasNEON())) {
9858 return fastEmitInst_rr(AArch64::SMINv2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9859 }
9860 return 0;
9861}
9862
9863unsigned fastEmit_ISD_SMIN_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9864 if (RetVT.SimpleTy != MVT::v4i32)
9865 return 0;
9866 if ((Subtarget->hasNEON())) {
9867 return fastEmitInst_rr(AArch64::SMINv4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9868 }
9869 return 0;
9870}
9871
9872unsigned fastEmit_ISD_SMIN_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9873 switch (VT.SimpleTy) {
9874 case MVT::v8i8: return fastEmit_ISD_SMIN_MVT_v8i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9875 case MVT::v16i8: return fastEmit_ISD_SMIN_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9876 case MVT::v4i16: return fastEmit_ISD_SMIN_MVT_v4i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9877 case MVT::v8i16: return fastEmit_ISD_SMIN_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9878 case MVT::v2i32: return fastEmit_ISD_SMIN_MVT_v2i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9879 case MVT::v4i32: return fastEmit_ISD_SMIN_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9880 default: return 0;
9881 }
9882}
9883
9884// FastEmit functions for ISD::SRA.
9885
9886unsigned fastEmit_ISD_SRA_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9887 if (RetVT.SimpleTy != MVT::i64)
9888 return 0;
9889 return fastEmitInst_rr(AArch64::ASRVXr, &AArch64::GPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9890}
9891
9892unsigned fastEmit_ISD_SRA_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9893 switch (VT.SimpleTy) {
9894 case MVT::i64: return fastEmit_ISD_SRA_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9895 default: return 0;
9896 }
9897}
9898
9899// FastEmit functions for ISD::SRL.
9900
9901unsigned fastEmit_ISD_SRL_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9902 if (RetVT.SimpleTy != MVT::i64)
9903 return 0;
9904 return fastEmitInst_rr(AArch64::LSRVXr, &AArch64::GPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9905}
9906
9907unsigned fastEmit_ISD_SRL_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9908 switch (VT.SimpleTy) {
9909 case MVT::i64: return fastEmit_ISD_SRL_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9910 default: return 0;
9911 }
9912}
9913
9914// FastEmit functions for ISD::SSUBSAT.
9915
9916unsigned fastEmit_ISD_SSUBSAT_MVT_v8i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9917 if (RetVT.SimpleTy != MVT::v8i8)
9918 return 0;
9919 if ((Subtarget->hasNEON())) {
9920 return fastEmitInst_rr(AArch64::SQSUBv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9921 }
9922 return 0;
9923}
9924
9925unsigned fastEmit_ISD_SSUBSAT_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9926 if (RetVT.SimpleTy != MVT::v16i8)
9927 return 0;
9928 if ((Subtarget->hasNEON())) {
9929 return fastEmitInst_rr(AArch64::SQSUBv16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9930 }
9931 return 0;
9932}
9933
9934unsigned fastEmit_ISD_SSUBSAT_MVT_v4i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9935 if (RetVT.SimpleTy != MVT::v4i16)
9936 return 0;
9937 if ((Subtarget->hasNEON())) {
9938 return fastEmitInst_rr(AArch64::SQSUBv4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9939 }
9940 return 0;
9941}
9942
9943unsigned fastEmit_ISD_SSUBSAT_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9944 if (RetVT.SimpleTy != MVT::v8i16)
9945 return 0;
9946 if ((Subtarget->hasNEON())) {
9947 return fastEmitInst_rr(AArch64::SQSUBv8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9948 }
9949 return 0;
9950}
9951
9952unsigned fastEmit_ISD_SSUBSAT_MVT_v2i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9953 if (RetVT.SimpleTy != MVT::v2i32)
9954 return 0;
9955 if ((Subtarget->hasNEON())) {
9956 return fastEmitInst_rr(AArch64::SQSUBv2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9957 }
9958 return 0;
9959}
9960
9961unsigned fastEmit_ISD_SSUBSAT_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9962 if (RetVT.SimpleTy != MVT::v4i32)
9963 return 0;
9964 if ((Subtarget->hasNEON())) {
9965 return fastEmitInst_rr(AArch64::SQSUBv4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9966 }
9967 return 0;
9968}
9969
9970unsigned fastEmit_ISD_SSUBSAT_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9971 if (RetVT.SimpleTy != MVT::v2i64)
9972 return 0;
9973 if ((Subtarget->hasNEON())) {
9974 return fastEmitInst_rr(AArch64::SQSUBv2i64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9975 }
9976 return 0;
9977}
9978
9979unsigned fastEmit_ISD_SSUBSAT_MVT_nxv16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9980 if (RetVT.SimpleTy != MVT::nxv16i8)
9981 return 0;
9982 if ((Subtarget->hasSVE())) {
9983 return fastEmitInst_rr(AArch64::SQSUB_ZZZ_B, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9984 }
9985 return 0;
9986}
9987
9988unsigned fastEmit_ISD_SSUBSAT_MVT_nxv8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9989 if (RetVT.SimpleTy != MVT::nxv8i16)
9990 return 0;
9991 if ((Subtarget->hasSVE())) {
9992 return fastEmitInst_rr(AArch64::SQSUB_ZZZ_H, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9993 }
9994 return 0;
9995}
9996
9997unsigned fastEmit_ISD_SSUBSAT_MVT_nxv4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9998 if (RetVT.SimpleTy != MVT::nxv4i32)
9999 return 0;
10000 if ((Subtarget->hasSVE())) {
10001 return fastEmitInst_rr(AArch64::SQSUB_ZZZ_S, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10002 }
10003 return 0;
10004}
10005
10006unsigned fastEmit_ISD_SSUBSAT_MVT_nxv2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10007 if (RetVT.SimpleTy != MVT::nxv2i64)
10008 return 0;
10009 if ((Subtarget->hasSVE())) {
10010 return fastEmitInst_rr(AArch64::SQSUB_ZZZ_D, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10011 }
10012 return 0;
10013}
10014
10015unsigned fastEmit_ISD_SSUBSAT_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10016 switch (VT.SimpleTy) {
10017 case MVT::v8i8: return fastEmit_ISD_SSUBSAT_MVT_v8i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10018 case MVT::v16i8: return fastEmit_ISD_SSUBSAT_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10019 case MVT::v4i16: return fastEmit_ISD_SSUBSAT_MVT_v4i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10020 case MVT::v8i16: return fastEmit_ISD_SSUBSAT_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10021 case MVT::v2i32: return fastEmit_ISD_SSUBSAT_MVT_v2i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10022 case MVT::v4i32: return fastEmit_ISD_SSUBSAT_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10023 case MVT::v2i64: return fastEmit_ISD_SSUBSAT_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10024 case MVT::nxv16i8: return fastEmit_ISD_SSUBSAT_MVT_nxv16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10025 case MVT::nxv8i16: return fastEmit_ISD_SSUBSAT_MVT_nxv8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10026 case MVT::nxv4i32: return fastEmit_ISD_SSUBSAT_MVT_nxv4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10027 case MVT::nxv2i64: return fastEmit_ISD_SSUBSAT_MVT_nxv2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10028 default: return 0;
10029 }
10030}
10031
10032// FastEmit functions for ISD::SUB.
10033
10034unsigned fastEmit_ISD_SUB_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10035 if (RetVT.SimpleTy != MVT::i32)
10036 return 0;
10037 return fastEmitInst_rr(AArch64::SUBSWrr, &AArch64::GPR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10038}
10039
10040unsigned fastEmit_ISD_SUB_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10041 if (RetVT.SimpleTy != MVT::i64)
10042 return 0;
10043 return fastEmitInst_rr(AArch64::SUBSXrr, &AArch64::GPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10044}
10045
10046unsigned fastEmit_ISD_SUB_MVT_v8i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10047 if (RetVT.SimpleTy != MVT::v8i8)
10048 return 0;
10049 if ((Subtarget->hasNEON())) {
10050 return fastEmitInst_rr(AArch64::SUBv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10051 }
10052 return 0;
10053}
10054
10055unsigned fastEmit_ISD_SUB_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10056 if (RetVT.SimpleTy != MVT::v16i8)
10057 return 0;
10058 if ((Subtarget->hasNEON())) {
10059 return fastEmitInst_rr(AArch64::SUBv16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10060 }
10061 return 0;
10062}
10063
10064unsigned fastEmit_ISD_SUB_MVT_v4i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10065 if (RetVT.SimpleTy != MVT::v4i16)
10066 return 0;
10067 if ((Subtarget->hasNEON())) {
10068 return fastEmitInst_rr(AArch64::SUBv4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10069 }
10070 return 0;
10071}
10072
10073unsigned fastEmit_ISD_SUB_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10074 if (RetVT.SimpleTy != MVT::v8i16)
10075 return 0;
10076 if ((Subtarget->hasNEON())) {
10077 return fastEmitInst_rr(AArch64::SUBv8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10078 }
10079 return 0;
10080}
10081
10082unsigned fastEmit_ISD_SUB_MVT_v2i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10083 if (RetVT.SimpleTy != MVT::v2i32)
10084 return 0;
10085 if ((Subtarget->hasNEON())) {
10086 return fastEmitInst_rr(AArch64::SUBv2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10087 }
10088 return 0;
10089}
10090
10091unsigned fastEmit_ISD_SUB_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10092 if (RetVT.SimpleTy != MVT::v4i32)
10093 return 0;
10094 if ((Subtarget->hasNEON())) {
10095 return fastEmitInst_rr(AArch64::SUBv4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10096 }
10097 return 0;
10098}
10099
10100unsigned fastEmit_ISD_SUB_MVT_v1i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10101 if (RetVT.SimpleTy != MVT::v1i64)
10102 return 0;
10103 if ((Subtarget->hasNEON())) {
10104 return fastEmitInst_rr(AArch64::SUBv1i64, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10105 }
10106 return 0;
10107}
10108
10109unsigned fastEmit_ISD_SUB_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10110 if (RetVT.SimpleTy != MVT::v2i64)
10111 return 0;
10112 if ((Subtarget->hasNEON())) {
10113 return fastEmitInst_rr(AArch64::SUBv2i64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10114 }
10115 return 0;
10116}
10117
10118unsigned fastEmit_ISD_SUB_MVT_nxv16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10119 if (RetVT.SimpleTy != MVT::nxv16i8)
10120 return 0;
10121 if ((Subtarget->hasSVE())) {
10122 return fastEmitInst_rr(AArch64::SUB_ZZZ_B, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10123 }
10124 return 0;
10125}
10126
10127unsigned fastEmit_ISD_SUB_MVT_nxv8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10128 if (RetVT.SimpleTy != MVT::nxv8i16)
10129 return 0;
10130 if ((Subtarget->hasSVE())) {
10131 return fastEmitInst_rr(AArch64::SUB_ZZZ_H, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10132 }
10133 return 0;
10134}
10135
10136unsigned fastEmit_ISD_SUB_MVT_nxv4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10137 if (RetVT.SimpleTy != MVT::nxv4i32)
10138 return 0;
10139 if ((Subtarget->hasSVE())) {
10140 return fastEmitInst_rr(AArch64::SUB_ZZZ_S, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10141 }
10142 return 0;
10143}
10144
10145unsigned fastEmit_ISD_SUB_MVT_nxv2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10146 if (RetVT.SimpleTy != MVT::nxv2i64)
10147 return 0;
10148 if ((Subtarget->hasSVE())) {
10149 return fastEmitInst_rr(AArch64::SUB_ZZZ_D, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10150 }
10151 return 0;
10152}
10153
10154unsigned fastEmit_ISD_SUB_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10155 switch (VT.SimpleTy) {
10156 case MVT::i32: return fastEmit_ISD_SUB_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10157 case MVT::i64: return fastEmit_ISD_SUB_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10158 case MVT::v8i8: return fastEmit_ISD_SUB_MVT_v8i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10159 case MVT::v16i8: return fastEmit_ISD_SUB_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10160 case MVT::v4i16: return fastEmit_ISD_SUB_MVT_v4i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10161 case MVT::v8i16: return fastEmit_ISD_SUB_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10162 case MVT::v2i32: return fastEmit_ISD_SUB_MVT_v2i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10163 case MVT::v4i32: return fastEmit_ISD_SUB_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10164 case MVT::v1i64: return fastEmit_ISD_SUB_MVT_v1i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10165 case MVT::v2i64: return fastEmit_ISD_SUB_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10166 case MVT::nxv16i8: return fastEmit_ISD_SUB_MVT_nxv16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10167 case MVT::nxv8i16: return fastEmit_ISD_SUB_MVT_nxv8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10168 case MVT::nxv4i32: return fastEmit_ISD_SUB_MVT_nxv4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10169 case MVT::nxv2i64: return fastEmit_ISD_SUB_MVT_nxv2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10170 default: return 0;
10171 }
10172}
10173
10174// FastEmit functions for ISD::UADDSAT.
10175
10176unsigned fastEmit_ISD_UADDSAT_MVT_v8i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10177 if (RetVT.SimpleTy != MVT::v8i8)
10178 return 0;
10179 if ((Subtarget->hasNEON())) {
10180 return fastEmitInst_rr(AArch64::UQADDv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10181 }
10182 return 0;
10183}
10184
10185unsigned fastEmit_ISD_UADDSAT_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10186 if (RetVT.SimpleTy != MVT::v16i8)
10187 return 0;
10188 if ((Subtarget->hasNEON())) {
10189 return fastEmitInst_rr(AArch64::UQADDv16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10190 }
10191 return 0;
10192}
10193
10194unsigned fastEmit_ISD_UADDSAT_MVT_v4i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10195 if (RetVT.SimpleTy != MVT::v4i16)
10196 return 0;
10197 if ((Subtarget->hasNEON())) {
10198 return fastEmitInst_rr(AArch64::UQADDv4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10199 }
10200 return 0;
10201}
10202
10203unsigned fastEmit_ISD_UADDSAT_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10204 if (RetVT.SimpleTy != MVT::v8i16)
10205 return 0;
10206 if ((Subtarget->hasNEON())) {
10207 return fastEmitInst_rr(AArch64::UQADDv8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10208 }
10209 return 0;
10210}
10211
10212unsigned fastEmit_ISD_UADDSAT_MVT_v2i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10213 if (RetVT.SimpleTy != MVT::v2i32)
10214 return 0;
10215 if ((Subtarget->hasNEON())) {
10216 return fastEmitInst_rr(AArch64::UQADDv2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10217 }
10218 return 0;
10219}
10220
10221unsigned fastEmit_ISD_UADDSAT_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10222 if (RetVT.SimpleTy != MVT::v4i32)
10223 return 0;
10224 if ((Subtarget->hasNEON())) {
10225 return fastEmitInst_rr(AArch64::UQADDv4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10226 }
10227 return 0;
10228}
10229
10230unsigned fastEmit_ISD_UADDSAT_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10231 if (RetVT.SimpleTy != MVT::v2i64)
10232 return 0;
10233 if ((Subtarget->hasNEON())) {
10234 return fastEmitInst_rr(AArch64::UQADDv2i64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10235 }
10236 return 0;
10237}
10238
10239unsigned fastEmit_ISD_UADDSAT_MVT_nxv16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10240 if (RetVT.SimpleTy != MVT::nxv16i8)
10241 return 0;
10242 if ((Subtarget->hasSVE())) {
10243 return fastEmitInst_rr(AArch64::UQADD_ZZZ_B, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10244 }
10245 return 0;
10246}
10247
10248unsigned fastEmit_ISD_UADDSAT_MVT_nxv8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10249 if (RetVT.SimpleTy != MVT::nxv8i16)
10250 return 0;
10251 if ((Subtarget->hasSVE())) {
10252 return fastEmitInst_rr(AArch64::UQADD_ZZZ_H, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10253 }
10254 return 0;
10255}
10256
10257unsigned fastEmit_ISD_UADDSAT_MVT_nxv4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10258 if (RetVT.SimpleTy != MVT::nxv4i32)
10259 return 0;
10260 if ((Subtarget->hasSVE())) {
10261 return fastEmitInst_rr(AArch64::UQADD_ZZZ_S, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10262 }
10263 return 0;
10264}
10265
10266unsigned fastEmit_ISD_UADDSAT_MVT_nxv2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10267 if (RetVT.SimpleTy != MVT::nxv2i64)
10268 return 0;
10269 if ((Subtarget->hasSVE())) {
10270 return fastEmitInst_rr(AArch64::UQADD_ZZZ_D, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10271 }
10272 return 0;
10273}
10274
10275unsigned fastEmit_ISD_UADDSAT_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10276 switch (VT.SimpleTy) {
10277 case MVT::v8i8: return fastEmit_ISD_UADDSAT_MVT_v8i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10278 case MVT::v16i8: return fastEmit_ISD_UADDSAT_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10279 case MVT::v4i16: return fastEmit_ISD_UADDSAT_MVT_v4i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10280 case MVT::v8i16: return fastEmit_ISD_UADDSAT_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10281 case MVT::v2i32: return fastEmit_ISD_UADDSAT_MVT_v2i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10282 case MVT::v4i32: return fastEmit_ISD_UADDSAT_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10283 case MVT::v2i64: return fastEmit_ISD_UADDSAT_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10284 case MVT::nxv16i8: return fastEmit_ISD_UADDSAT_MVT_nxv16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10285 case MVT::nxv8i16: return fastEmit_ISD_UADDSAT_MVT_nxv8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10286 case MVT::nxv4i32: return fastEmit_ISD_UADDSAT_MVT_nxv4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10287 case MVT::nxv2i64: return fastEmit_ISD_UADDSAT_MVT_nxv2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10288 default: return 0;
10289 }
10290}
10291
10292// FastEmit functions for ISD::UDIV.
10293
10294unsigned fastEmit_ISD_UDIV_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10295 if (RetVT.SimpleTy != MVT::i32)
10296 return 0;
10297 return fastEmitInst_rr(AArch64::UDIVWr, &AArch64::GPR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10298}
10299
10300unsigned fastEmit_ISD_UDIV_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10301 if (RetVT.SimpleTy != MVT::i64)
10302 return 0;
10303 return fastEmitInst_rr(AArch64::UDIVXr, &AArch64::GPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10304}
10305
10306unsigned fastEmit_ISD_UDIV_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10307 switch (VT.SimpleTy) {
10308 case MVT::i32: return fastEmit_ISD_UDIV_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10309 case MVT::i64: return fastEmit_ISD_UDIV_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10310 default: return 0;
10311 }
10312}
10313
10314// FastEmit functions for ISD::UMAX.
10315
10316unsigned fastEmit_ISD_UMAX_MVT_v8i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10317 if (RetVT.SimpleTy != MVT::v8i8)
10318 return 0;
10319 if ((Subtarget->hasNEON())) {
10320 return fastEmitInst_rr(AArch64::UMAXv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10321 }
10322 return 0;
10323}
10324
10325unsigned fastEmit_ISD_UMAX_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10326 if (RetVT.SimpleTy != MVT::v16i8)
10327 return 0;
10328 if ((Subtarget->hasNEON())) {
10329 return fastEmitInst_rr(AArch64::UMAXv16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10330 }
10331 return 0;
10332}
10333
10334unsigned fastEmit_ISD_UMAX_MVT_v4i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10335 if (RetVT.SimpleTy != MVT::v4i16)
10336 return 0;
10337 if ((Subtarget->hasNEON())) {
10338 return fastEmitInst_rr(AArch64::UMAXv4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10339 }
10340 return 0;
10341}
10342
10343unsigned fastEmit_ISD_UMAX_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10344 if (RetVT.SimpleTy != MVT::v8i16)
10345 return 0;
10346 if ((Subtarget->hasNEON())) {
10347 return fastEmitInst_rr(AArch64::UMAXv8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10348 }
10349 return 0;
10350}
10351
10352unsigned fastEmit_ISD_UMAX_MVT_v2i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10353 if (RetVT.SimpleTy != MVT::v2i32)
10354 return 0;
10355 if ((Subtarget->hasNEON())) {
10356 return fastEmitInst_rr(AArch64::UMAXv2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10357 }
10358 return 0;
10359}
10360
10361unsigned fastEmit_ISD_UMAX_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10362 if (RetVT.SimpleTy != MVT::v4i32)
10363 return 0;
10364 if ((Subtarget->hasNEON())) {
10365 return fastEmitInst_rr(AArch64::UMAXv4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10366 }
10367 return 0;
10368}
10369
10370unsigned fastEmit_ISD_UMAX_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10371 switch (VT.SimpleTy) {
10372 case MVT::v8i8: return fastEmit_ISD_UMAX_MVT_v8i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10373 case MVT::v16i8: return fastEmit_ISD_UMAX_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10374 case MVT::v4i16: return fastEmit_ISD_UMAX_MVT_v4i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10375 case MVT::v8i16: return fastEmit_ISD_UMAX_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10376 case MVT::v2i32: return fastEmit_ISD_UMAX_MVT_v2i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10377 case MVT::v4i32: return fastEmit_ISD_UMAX_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10378 default: return 0;
10379 }
10380}
10381
10382// FastEmit functions for ISD::UMIN.
10383
10384unsigned fastEmit_ISD_UMIN_MVT_v8i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10385 if (RetVT.SimpleTy != MVT::v8i8)
10386 return 0;
10387 if ((Subtarget->hasNEON())) {
10388 return fastEmitInst_rr(AArch64::UMINv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10389 }
10390 return 0;
10391}
10392
10393unsigned fastEmit_ISD_UMIN_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10394 if (RetVT.SimpleTy != MVT::v16i8)
10395 return 0;
10396 if ((Subtarget->hasNEON())) {
10397 return fastEmitInst_rr(AArch64::UMINv16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10398 }
10399 return 0;
10400}
10401
10402unsigned fastEmit_ISD_UMIN_MVT_v4i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10403 if (RetVT.SimpleTy != MVT::v4i16)
10404 return 0;
10405 if ((Subtarget->hasNEON())) {
10406 return fastEmitInst_rr(AArch64::UMINv4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10407 }
10408 return 0;
10409}
10410
10411unsigned fastEmit_ISD_UMIN_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10412 if (RetVT.SimpleTy != MVT::v8i16)
10413 return 0;
10414 if ((Subtarget->hasNEON())) {
10415 return fastEmitInst_rr(AArch64::UMINv8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10416 }
10417 return 0;
10418}
10419
10420unsigned fastEmit_ISD_UMIN_MVT_v2i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10421 if (RetVT.SimpleTy != MVT::v2i32)
10422 return 0;
10423 if ((Subtarget->hasNEON())) {
10424 return fastEmitInst_rr(AArch64::UMINv2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10425 }
10426 return 0;
10427}
10428
10429unsigned fastEmit_ISD_UMIN_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10430 if (RetVT.SimpleTy != MVT::v4i32)
10431 return 0;
10432 if ((Subtarget->hasNEON())) {
10433 return fastEmitInst_rr(AArch64::UMINv4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10434 }
10435 return 0;
10436}
10437
10438unsigned fastEmit_ISD_UMIN_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10439 switch (VT.SimpleTy) {
10440 case MVT::v8i8: return fastEmit_ISD_UMIN_MVT_v8i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10441 case MVT::v16i8: return fastEmit_ISD_UMIN_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10442 case MVT::v4i16: return fastEmit_ISD_UMIN_MVT_v4i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10443 case MVT::v8i16: return fastEmit_ISD_UMIN_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10444 case MVT::v2i32: return fastEmit_ISD_UMIN_MVT_v2i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10445 case MVT::v4i32: return fastEmit_ISD_UMIN_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10446 default: return 0;
10447 }
10448}
10449
10450// FastEmit functions for ISD::USUBSAT.
10451
10452unsigned fastEmit_ISD_USUBSAT_MVT_v8i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10453 if (RetVT.SimpleTy != MVT::v8i8)
10454 return 0;
10455 if ((Subtarget->hasNEON())) {
10456 return fastEmitInst_rr(AArch64::UQSUBv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10457 }
10458 return 0;
10459}
10460
10461unsigned fastEmit_ISD_USUBSAT_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10462 if (RetVT.SimpleTy != MVT::v16i8)
10463 return 0;
10464 if ((Subtarget->hasNEON())) {
10465 return fastEmitInst_rr(AArch64::UQSUBv16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10466 }
10467 return 0;
10468}
10469
10470unsigned fastEmit_ISD_USUBSAT_MVT_v4i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10471 if (RetVT.SimpleTy != MVT::v4i16)
10472 return 0;
10473 if ((Subtarget->hasNEON())) {
10474 return fastEmitInst_rr(AArch64::UQSUBv4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10475 }
10476 return 0;
10477}
10478
10479unsigned fastEmit_ISD_USUBSAT_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10480 if (RetVT.SimpleTy != MVT::v8i16)
10481 return 0;
10482 if ((Subtarget->hasNEON())) {
10483 return fastEmitInst_rr(AArch64::UQSUBv8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10484 }
10485 return 0;
10486}
10487
10488unsigned fastEmit_ISD_USUBSAT_MVT_v2i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10489 if (RetVT.SimpleTy != MVT::v2i32)
10490 return 0;
10491 if ((Subtarget->hasNEON())) {
10492 return fastEmitInst_rr(AArch64::UQSUBv2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10493 }
10494 return 0;
10495}
10496
10497unsigned fastEmit_ISD_USUBSAT_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10498 if (RetVT.SimpleTy != MVT::v4i32)
10499 return 0;
10500 if ((Subtarget->hasNEON())) {
10501 return fastEmitInst_rr(AArch64::UQSUBv4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10502 }
10503 return 0;
10504}
10505
10506unsigned fastEmit_ISD_USUBSAT_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10507 if (RetVT.SimpleTy != MVT::v2i64)
10508 return 0;
10509 if ((Subtarget->hasNEON())) {
10510 return fastEmitInst_rr(AArch64::UQSUBv2i64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10511 }
10512 return 0;
10513}
10514
10515unsigned fastEmit_ISD_USUBSAT_MVT_nxv16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10516 if (RetVT.SimpleTy != MVT::nxv16i8)
10517 return 0;
10518 if ((Subtarget->hasSVE())) {
10519 return fastEmitInst_rr(AArch64::UQSUB_ZZZ_B, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10520 }
10521 return 0;
10522}
10523
10524unsigned fastEmit_ISD_USUBSAT_MVT_nxv8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10525 if (RetVT.SimpleTy != MVT::nxv8i16)
10526 return 0;
10527 if ((Subtarget->hasSVE())) {
10528 return fastEmitInst_rr(AArch64::UQSUB_ZZZ_H, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10529 }
10530 return 0;
10531}
10532
10533unsigned fastEmit_ISD_USUBSAT_MVT_nxv4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10534 if (RetVT.SimpleTy != MVT::nxv4i32)
10535 return 0;
10536 if ((Subtarget->hasSVE())) {
10537 return fastEmitInst_rr(AArch64::UQSUB_ZZZ_S, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10538 }
10539 return 0;
10540}
10541
10542unsigned fastEmit_ISD_USUBSAT_MVT_nxv2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10543 if (RetVT.SimpleTy != MVT::nxv2i64)
10544 return 0;
10545 if ((Subtarget->hasSVE())) {
10546 return fastEmitInst_rr(AArch64::UQSUB_ZZZ_D, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10547 }
10548 return 0;
10549}
10550
10551unsigned fastEmit_ISD_USUBSAT_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10552 switch (VT.SimpleTy) {
10553 case MVT::v8i8: return fastEmit_ISD_USUBSAT_MVT_v8i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10554 case MVT::v16i8: return fastEmit_ISD_USUBSAT_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10555 case MVT::v4i16: return fastEmit_ISD_USUBSAT_MVT_v4i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10556 case MVT::v8i16: return fastEmit_ISD_USUBSAT_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10557 case MVT::v2i32: return fastEmit_ISD_USUBSAT_MVT_v2i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10558 case MVT::v4i32: return fastEmit_ISD_USUBSAT_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10559 case MVT::v2i64: return fastEmit_ISD_USUBSAT_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10560 case MVT::nxv16i8: return fastEmit_ISD_USUBSAT_MVT_nxv16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10561 case MVT::nxv8i16: return fastEmit_ISD_USUBSAT_MVT_nxv8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10562 case MVT::nxv4i32: return fastEmit_ISD_USUBSAT_MVT_nxv4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10563 case MVT::nxv2i64: return fastEmit_ISD_USUBSAT_MVT_nxv2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10564 default: return 0;
10565 }
10566}
10567
10568// FastEmit functions for ISD::XOR.
10569
10570unsigned fastEmit_ISD_XOR_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10571 if (RetVT.SimpleTy != MVT::i32)
10572 return 0;
10573 return fastEmitInst_rr(AArch64::EORWrr, &AArch64::GPR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10574}
10575
10576unsigned fastEmit_ISD_XOR_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10577 if (RetVT.SimpleTy != MVT::i64)
10578 return 0;
10579 return fastEmitInst_rr(AArch64::EORXrr, &AArch64::GPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10580}
10581
10582unsigned fastEmit_ISD_XOR_MVT_v8i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10583 if (RetVT.SimpleTy != MVT::v8i8)
10584 return 0;
10585 if ((Subtarget->hasNEON())) {
10586 return fastEmitInst_rr(AArch64::EORv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10587 }
10588 return 0;
10589}
10590
10591unsigned fastEmit_ISD_XOR_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10592 if (RetVT.SimpleTy != MVT::v16i8)
10593 return 0;
10594 if ((Subtarget->hasNEON())) {
10595 return fastEmitInst_rr(AArch64::EORv16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10596 }
10597 return 0;
10598}
10599
10600unsigned fastEmit_ISD_XOR_MVT_v4i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10601 if (RetVT.SimpleTy != MVT::v4i16)
10602 return 0;
10603 if ((Subtarget->hasNEON())) {
10604 return fastEmitInst_rr(AArch64::EORv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10605 }
10606 return 0;
10607}
10608
10609unsigned fastEmit_ISD_XOR_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10610 if (RetVT.SimpleTy != MVT::v8i16)
10611 return 0;
10612 if ((Subtarget->hasNEON())) {
10613 return fastEmitInst_rr(AArch64::EORv16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10614 }
10615 return 0;
10616}
10617
10618unsigned fastEmit_ISD_XOR_MVT_v2i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10619 if (RetVT.SimpleTy != MVT::v2i32)
10620 return 0;
10621 if ((Subtarget->hasNEON())) {
10622 return fastEmitInst_rr(AArch64::EORv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10623 }
10624 return 0;
10625}
10626
10627unsigned fastEmit_ISD_XOR_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10628 if (RetVT.SimpleTy != MVT::v4i32)
10629 return 0;
10630 if ((Subtarget->hasNEON())) {
10631 return fastEmitInst_rr(AArch64::EORv16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10632 }
10633 return 0;
10634}
10635
10636unsigned fastEmit_ISD_XOR_MVT_v1i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10637 if (RetVT.SimpleTy != MVT::v1i64)
10638 return 0;
10639 if ((Subtarget->hasNEON())) {
10640 return fastEmitInst_rr(AArch64::EORv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10641 }
10642 return 0;
10643}
10644
10645unsigned fastEmit_ISD_XOR_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10646 if (RetVT.SimpleTy != MVT::v2i64)
10647 return 0;
10648 if ((Subtarget->hasNEON())) {
10649 return fastEmitInst_rr(AArch64::EORv16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10650 }
10651 return 0;
10652}
10653
10654unsigned fastEmit_ISD_XOR_MVT_nxv16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10655 if (RetVT.SimpleTy != MVT::nxv16i8)
10656 return 0;
10657 if ((Subtarget->hasSVE())) {
10658 return fastEmitInst_rr(AArch64::EOR_ZZZ, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10659 }
10660 return 0;
10661}
10662
10663unsigned fastEmit_ISD_XOR_MVT_nxv8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10664 if (RetVT.SimpleTy != MVT::nxv8i16)
10665 return 0;
10666 if ((Subtarget->hasSVE())) {
10667 return fastEmitInst_rr(AArch64::EOR_ZZZ, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10668 }
10669 return 0;
10670}
10671
10672unsigned fastEmit_ISD_XOR_MVT_nxv4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10673 if (RetVT.SimpleTy != MVT::nxv4i32)
10674 return 0;
10675 if ((Subtarget->hasSVE())) {
10676 return fastEmitInst_rr(AArch64::EOR_ZZZ, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10677 }
10678 return 0;
10679}
10680
10681unsigned fastEmit_ISD_XOR_MVT_nxv2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10682 if (RetVT.SimpleTy != MVT::nxv2i64)
10683 return 0;
10684 if ((Subtarget->hasSVE())) {
10685 return fastEmitInst_rr(AArch64::EOR_ZZZ, &AArch64::ZPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10686 }
10687 return 0;
10688}
10689
10690unsigned fastEmit_ISD_XOR_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10691 switch (VT.SimpleTy) {
10692 case MVT::i32: return fastEmit_ISD_XOR_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10693 case MVT::i64: return fastEmit_ISD_XOR_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10694 case MVT::v8i8: return fastEmit_ISD_XOR_MVT_v8i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10695 case MVT::v16i8: return fastEmit_ISD_XOR_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10696 case MVT::v4i16: return fastEmit_ISD_XOR_MVT_v4i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10697 case MVT::v8i16: return fastEmit_ISD_XOR_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10698 case MVT::v2i32: return fastEmit_ISD_XOR_MVT_v2i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10699 case MVT::v4i32: return fastEmit_ISD_XOR_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10700 case MVT::v1i64: return fastEmit_ISD_XOR_MVT_v1i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10701 case MVT::v2i64: return fastEmit_ISD_XOR_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10702 case MVT::nxv16i8: return fastEmit_ISD_XOR_MVT_nxv16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10703 case MVT::nxv8i16: return fastEmit_ISD_XOR_MVT_nxv8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10704 case MVT::nxv4i32: return fastEmit_ISD_XOR_MVT_nxv4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10705 case MVT::nxv2i64: return fastEmit_ISD_XOR_MVT_nxv2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10706 default: return 0;
10707 }
10708}
10709
10710// Top-level FastEmit function.
10711
10712unsigned fastEmit_rr(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) override {
10713 switch (Opcode) {
10714 case AArch64ISD::CMEQ: return fastEmit_AArch64ISD_CMEQ_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10715 case AArch64ISD::CMGE: return fastEmit_AArch64ISD_CMGE_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10716 case AArch64ISD::CMGT: return fastEmit_AArch64ISD_CMGT_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10717 case AArch64ISD::CMHI: return fastEmit_AArch64ISD_CMHI_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10718 case AArch64ISD::CMHS: return fastEmit_AArch64ISD_CMHS_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10719 case AArch64ISD::FCMEQ: return fastEmit_AArch64ISD_FCMEQ_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10720 case AArch64ISD::FCMGE: return fastEmit_AArch64ISD_FCMGE_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10721 case AArch64ISD::FCMGT: return fastEmit_AArch64ISD_FCMGT_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10722 case AArch64ISD::FCMP: return fastEmit_AArch64ISD_FCMP_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10723 case AArch64ISD::FRECPS: return fastEmit_AArch64ISD_FRECPS_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10724 case AArch64ISD::FRSQRTS: return fastEmit_AArch64ISD_FRSQRTS_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10725 case AArch64ISD::INDEX_VECTOR: return fastEmit_AArch64ISD_INDEX_VECTOR_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10726 case AArch64ISD::PTEST: return fastEmit_AArch64ISD_PTEST_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10727 case AArch64ISD::SABD: return fastEmit_AArch64ISD_SABD_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10728 case AArch64ISD::SHADD: return fastEmit_AArch64ISD_SHADD_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10729 case AArch64ISD::SMULL: return fastEmit_AArch64ISD_SMULL_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10730 case AArch64ISD::SRHADD: return fastEmit_AArch64ISD_SRHADD_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10731 case AArch64ISD::STRICT_FCMP: return fastEmit_AArch64ISD_STRICT_FCMP_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10732 case AArch64ISD::STRICT_FCMPE: return fastEmit_AArch64ISD_STRICT_FCMPE_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10733 case AArch64ISD::TBL: return fastEmit_AArch64ISD_TBL_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10734 case AArch64ISD::TRN1: return fastEmit_AArch64ISD_TRN1_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10735 case AArch64ISD::TRN2: return fastEmit_AArch64ISD_TRN2_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10736 case AArch64ISD::UABD: return fastEmit_AArch64ISD_UABD_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10737 case AArch64ISD::UHADD: return fastEmit_AArch64ISD_UHADD_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10738 case AArch64ISD::UMULL: return fastEmit_AArch64ISD_UMULL_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10739 case AArch64ISD::URHADD: return fastEmit_AArch64ISD_URHADD_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10740 case AArch64ISD::UZP1: return fastEmit_AArch64ISD_UZP1_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10741 case AArch64ISD::UZP2: return fastEmit_AArch64ISD_UZP2_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10742 case AArch64ISD::ZIP1: return fastEmit_AArch64ISD_ZIP1_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10743 case AArch64ISD::ZIP2: return fastEmit_AArch64ISD_ZIP2_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10744 case ISD::ADD: return fastEmit_ISD_ADD_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10745 case ISD::AND: return fastEmit_ISD_AND_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10746 case ISD::CONCAT_VECTORS: return fastEmit_ISD_CONCAT_VECTORS_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10747 case ISD::FADD: return fastEmit_ISD_FADD_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10748 case ISD::FDIV: return fastEmit_ISD_FDIV_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10749 case ISD::FMAXIMUM: return fastEmit_ISD_FMAXIMUM_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10750 case ISD::FMAXNUM: return fastEmit_ISD_FMAXNUM_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10751 case ISD::FMINIMUM: return fastEmit_ISD_FMINIMUM_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10752 case ISD::FMINNUM: return fastEmit_ISD_FMINNUM_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10753 case ISD::FMUL: return fastEmit_ISD_FMUL_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10754 case ISD::FSUB: return fastEmit_ISD_FSUB_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10755 case ISD::MUL: return fastEmit_ISD_MUL_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10756 case ISD::MULHS: return fastEmit_ISD_MULHS_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10757 case ISD::MULHU: return fastEmit_ISD_MULHU_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10758 case ISD::OR: return fastEmit_ISD_OR_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10759 case ISD::ROTR: return fastEmit_ISD_ROTR_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10760 case ISD::SADDSAT: return fastEmit_ISD_SADDSAT_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10761 case ISD::SDIV: return fastEmit_ISD_SDIV_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10762 case ISD::SHL: return fastEmit_ISD_SHL_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10763 case ISD::SMAX: return fastEmit_ISD_SMAX_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10764 case ISD::SMIN: return fastEmit_ISD_SMIN_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10765 case ISD::SRA: return fastEmit_ISD_SRA_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10766 case ISD::SRL: return fastEmit_ISD_SRL_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10767 case ISD::SSUBSAT: return fastEmit_ISD_SSUBSAT_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10768 case ISD::SUB: return fastEmit_ISD_SUB_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10769 case ISD::UADDSAT: return fastEmit_ISD_UADDSAT_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10770 case ISD::UDIV: return fastEmit_ISD_UDIV_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10771 case ISD::UMAX: return fastEmit_ISD_UMAX_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10772 case ISD::UMIN: return fastEmit_ISD_UMIN_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10773 case ISD::USUBSAT: return fastEmit_ISD_USUBSAT_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10774 case ISD::XOR: return fastEmit_ISD_XOR_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10775 default: return 0;
10776 }
10777}
10778
10779// FastEmit functions for AArch64ISD::DUPLANE64.
10780
10781unsigned fastEmit_AArch64ISD_DUPLANE64_MVT_v2i64_ri_Predicate_VectorIndexD(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
10782 if (RetVT.SimpleTy != MVT::v2i64)
10783 return 0;
10784 if ((Subtarget->hasNEON())) {
10785 return fastEmitInst_ri(AArch64::DUPv2i64lane, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
10786 }
10787 return 0;
10788}
10789
10790unsigned fastEmit_AArch64ISD_DUPLANE64_MVT_v2f64_ri_Predicate_VectorIndexD(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
10791 if (RetVT.SimpleTy != MVT::v2f64)
10792 return 0;
10793 return fastEmitInst_ri(AArch64::DUPv2i64lane, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
10794}
10795
10796unsigned fastEmit_AArch64ISD_DUPLANE64_ri_Predicate_VectorIndexD(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
10797 switch (VT.SimpleTy) {
10798 case MVT::v2i64: return fastEmit_AArch64ISD_DUPLANE64_MVT_v2i64_ri_Predicate_VectorIndexD(RetVT, Op0, Op0IsKill, imm1);
10799 case MVT::v2f64: return fastEmit_AArch64ISD_DUPLANE64_MVT_v2f64_ri_Predicate_VectorIndexD(RetVT, Op0, Op0IsKill, imm1);
10800 default: return 0;
10801 }
10802}
10803
10804// FastEmit functions for ISD::EXTRACT_VECTOR_ELT.
10805
10806unsigned fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2i64_ri_Predicate_VectorIndexD(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
10807 if (RetVT.SimpleTy != MVT::i64)
10808 return 0;
10809 if ((Subtarget->hasNEON())) {
10810 return fastEmitInst_ri(AArch64::UMOVvi64, &AArch64::GPR64RegClass, Op0, Op0IsKill, imm1);
10811 }
10812 return 0;
10813}
10814
10815unsigned fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2f64_ri_Predicate_VectorIndexD(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
10816 if (RetVT.SimpleTy != MVT::f64)
10817 return 0;
10818 return fastEmitInst_ri(AArch64::CPYi64, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
10819}
10820
10821unsigned fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexD(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
10822 switch (VT.SimpleTy) {
10823 case MVT::v2i64: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2i64_ri_Predicate_VectorIndexD(RetVT, Op0, Op0IsKill, imm1);
10824 case MVT::v2f64: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2f64_ri_Predicate_VectorIndexD(RetVT, Op0, Op0IsKill, imm1);
10825 default: return 0;
10826 }
10827}
10828
10829// Top-level FastEmit function.
10830
10831unsigned fastEmit_ri_Predicate_VectorIndexD(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
10832 switch (Opcode) {
10833 case AArch64ISD::DUPLANE64: return fastEmit_AArch64ISD_DUPLANE64_ri_Predicate_VectorIndexD(VT, RetVT, Op0, Op0IsKill, imm1);
10834 case ISD::EXTRACT_VECTOR_ELT: return fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexD(VT, RetVT, Op0, Op0IsKill, imm1);
10835 default: return 0;
10836 }
10837}
10838
10839// FastEmit functions for AArch64ISD::DUPLANE32.
10840
10841unsigned fastEmit_AArch64ISD_DUPLANE32_MVT_v4i32_MVT_v2i32_ri_Predicate_VectorIndexS(unsigned Op0, bool Op0IsKill, uint64_t imm1) {
10842 if ((Subtarget->hasNEON())) {
10843 return fastEmitInst_ri(AArch64::DUPv2i32lane, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
10844 }
10845 return 0;
10846}
10847
10848unsigned fastEmit_AArch64ISD_DUPLANE32_MVT_v4i32_MVT_v4i32_ri_Predicate_VectorIndexS(unsigned Op0, bool Op0IsKill, uint64_t imm1) {
10849 if ((Subtarget->hasNEON())) {
10850 return fastEmitInst_ri(AArch64::DUPv4i32lane, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
10851 }
10852 return 0;
10853}
10854
10855unsigned fastEmit_AArch64ISD_DUPLANE32_MVT_v4i32_ri_Predicate_VectorIndexS(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
10856switch (RetVT.SimpleTy) {
10857 case MVT::v2i32: return fastEmit_AArch64ISD_DUPLANE32_MVT_v4i32_MVT_v2i32_ri_Predicate_VectorIndexS(Op0, Op0IsKill, imm1);
10858 case MVT::v4i32: return fastEmit_AArch64ISD_DUPLANE32_MVT_v4i32_MVT_v4i32_ri_Predicate_VectorIndexS(Op0, Op0IsKill, imm1);
10859 default: return 0;
10860}
10861}
10862
10863unsigned fastEmit_AArch64ISD_DUPLANE32_MVT_v4f32_MVT_v2f32_ri_Predicate_VectorIndexS(unsigned Op0, bool Op0IsKill, uint64_t imm1) {
10864 return fastEmitInst_ri(AArch64::DUPv2i32lane, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
10865}
10866
10867unsigned fastEmit_AArch64ISD_DUPLANE32_MVT_v4f32_MVT_v4f32_ri_Predicate_VectorIndexS(unsigned Op0, bool Op0IsKill, uint64_t imm1) {
10868 return fastEmitInst_ri(AArch64::DUPv4i32lane, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
10869}
10870
10871unsigned fastEmit_AArch64ISD_DUPLANE32_MVT_v4f32_ri_Predicate_VectorIndexS(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
10872switch (RetVT.SimpleTy) {
10873 case MVT::v2f32: return fastEmit_AArch64ISD_DUPLANE32_MVT_v4f32_MVT_v2f32_ri_Predicate_VectorIndexS(Op0, Op0IsKill, imm1);
10874 case MVT::v4f32: return fastEmit_AArch64ISD_DUPLANE32_MVT_v4f32_MVT_v4f32_ri_Predicate_VectorIndexS(Op0, Op0IsKill, imm1);
10875 default: return 0;
10876}
10877}
10878
10879unsigned fastEmit_AArch64ISD_DUPLANE32_ri_Predicate_VectorIndexS(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
10880 switch (VT.SimpleTy) {
10881 case MVT::v4i32: return fastEmit_AArch64ISD_DUPLANE32_MVT_v4i32_ri_Predicate_VectorIndexS(RetVT, Op0, Op0IsKill, imm1);
10882 case MVT::v4f32: return fastEmit_AArch64ISD_DUPLANE32_MVT_v4f32_ri_Predicate_VectorIndexS(RetVT, Op0, Op0IsKill, imm1);
10883 default: return 0;
10884 }
10885}
10886
10887// FastEmit functions for ISD::EXTRACT_VECTOR_ELT.
10888
10889unsigned fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4i32_ri_Predicate_VectorIndexS(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
10890 if (RetVT.SimpleTy != MVT::i32)
10891 return 0;
10892 if ((Subtarget->hasNEON())) {
10893 return fastEmitInst_ri(AArch64::UMOVvi32, &AArch64::GPR32RegClass, Op0, Op0IsKill, imm1);
10894 }
10895 return 0;
10896}
10897
10898unsigned fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4f32_ri_Predicate_VectorIndexS(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
10899 if (RetVT.SimpleTy != MVT::f32)
10900 return 0;
10901 return fastEmitInst_ri(AArch64::CPYi32, &AArch64::FPR32RegClass, Op0, Op0IsKill, imm1);
10902}
10903
10904unsigned fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexS(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
10905 switch (VT.SimpleTy) {
10906 case MVT::v4i32: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4i32_ri_Predicate_VectorIndexS(RetVT, Op0, Op0IsKill, imm1);
10907 case MVT::v4f32: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4f32_ri_Predicate_VectorIndexS(RetVT, Op0, Op0IsKill, imm1);
10908 default: return 0;
10909 }
10910}
10911
10912// Top-level FastEmit function.
10913
10914unsigned fastEmit_ri_Predicate_VectorIndexS(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
10915 switch (Opcode) {
10916 case AArch64ISD::DUPLANE32: return fastEmit_AArch64ISD_DUPLANE32_ri_Predicate_VectorIndexS(VT, RetVT, Op0, Op0IsKill, imm1);
10917 case ISD::EXTRACT_VECTOR_ELT: return fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexS(VT, RetVT, Op0, Op0IsKill, imm1);
10918 default: return 0;
10919 }
10920}
10921
10922// FastEmit functions for AArch64ISD::DUPLANE16.
10923
10924unsigned fastEmit_AArch64ISD_DUPLANE16_MVT_v8i16_MVT_v4i16_ri_Predicate_VectorIndexH(unsigned Op0, bool Op0IsKill, uint64_t imm1) {
10925 if ((Subtarget->hasNEON())) {
10926 return fastEmitInst_ri(AArch64::DUPv4i16lane, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
10927 }
10928 return 0;
10929}
10930
10931unsigned fastEmit_AArch64ISD_DUPLANE16_MVT_v8i16_MVT_v8i16_ri_Predicate_VectorIndexH(unsigned Op0, bool Op0IsKill, uint64_t imm1) {
10932 if ((Subtarget->hasNEON())) {
10933 return fastEmitInst_ri(AArch64::DUPv8i16lane, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
10934 }
10935 return 0;
10936}
10937
10938unsigned fastEmit_AArch64ISD_DUPLANE16_MVT_v8i16_ri_Predicate_VectorIndexH(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
10939switch (RetVT.SimpleTy) {
10940 case MVT::v4i16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8i16_MVT_v4i16_ri_Predicate_VectorIndexH(Op0, Op0IsKill, imm1);
10941 case MVT::v8i16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8i16_MVT_v8i16_ri_Predicate_VectorIndexH(Op0, Op0IsKill, imm1);
10942 default: return 0;
10943}
10944}
10945
10946unsigned fastEmit_AArch64ISD_DUPLANE16_MVT_v8f16_MVT_v4f16_ri_Predicate_VectorIndexH(unsigned Op0, bool Op0IsKill, uint64_t imm1) {
10947 return fastEmitInst_ri(AArch64::DUPv4i16lane, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
10948}
10949
10950unsigned fastEmit_AArch64ISD_DUPLANE16_MVT_v8f16_MVT_v8f16_ri_Predicate_VectorIndexH(unsigned Op0, bool Op0IsKill, uint64_t imm1) {
10951 return fastEmitInst_ri(AArch64::DUPv8i16lane, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
10952}
10953
10954unsigned fastEmit_AArch64ISD_DUPLANE16_MVT_v8f16_ri_Predicate_VectorIndexH(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
10955switch (RetVT.SimpleTy) {
10956 case MVT::v4f16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8f16_MVT_v4f16_ri_Predicate_VectorIndexH(Op0, Op0IsKill, imm1);
10957 case MVT::v8f16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8f16_MVT_v8f16_ri_Predicate_VectorIndexH(Op0, Op0IsKill, imm1);
10958 default: return 0;
10959}
10960}
10961
10962unsigned fastEmit_AArch64ISD_DUPLANE16_MVT_v8bf16_MVT_v4bf16_ri_Predicate_VectorIndexH(unsigned Op0, bool Op0IsKill, uint64_t imm1) {
10963 return fastEmitInst_ri(AArch64::DUPv4i16lane, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
10964}
10965
10966unsigned fastEmit_AArch64ISD_DUPLANE16_MVT_v8bf16_MVT_v8bf16_ri_Predicate_VectorIndexH(unsigned Op0, bool Op0IsKill, uint64_t imm1) {
10967 return fastEmitInst_ri(AArch64::DUPv8i16lane, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
10968}
10969
10970unsigned fastEmit_AArch64ISD_DUPLANE16_MVT_v8bf16_ri_Predicate_VectorIndexH(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
10971switch (RetVT.SimpleTy) {
10972 case MVT::v4bf16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8bf16_MVT_v4bf16_ri_Predicate_VectorIndexH(Op0, Op0IsKill, imm1);
10973 case MVT::v8bf16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8bf16_MVT_v8bf16_ri_Predicate_VectorIndexH(Op0, Op0IsKill, imm1);
10974 default: return 0;
10975}
10976}
10977
10978unsigned fastEmit_AArch64ISD_DUPLANE16_ri_Predicate_VectorIndexH(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
10979 switch (VT.SimpleTy) {
10980 case MVT::v8i16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8i16_ri_Predicate_VectorIndexH(RetVT, Op0, Op0IsKill, imm1);
10981 case MVT::v8f16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8f16_ri_Predicate_VectorIndexH(RetVT, Op0, Op0IsKill, imm1);
10982 case MVT::v8bf16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8bf16_ri_Predicate_VectorIndexH(RetVT, Op0, Op0IsKill, imm1);
10983 default: return 0;
10984 }
10985}
10986
10987// FastEmit functions for ISD::EXTRACT_VECTOR_ELT.
10988
10989unsigned fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8i16_ri_Predicate_VectorIndexH(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
10990 if (RetVT.SimpleTy != MVT::i32)
10991 return 0;
10992 if ((Subtarget->hasNEON())) {
10993 return fastEmitInst_ri(AArch64::UMOVvi16, &AArch64::GPR32RegClass, Op0, Op0IsKill, imm1);
10994 }
10995 return 0;
10996}
10997
10998unsigned fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8f16_ri_Predicate_VectorIndexH(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
10999 if (RetVT.SimpleTy != MVT::f16)
11000 return 0;
11001 return fastEmitInst_ri(AArch64::CPYi16, &AArch64::FPR16RegClass, Op0, Op0IsKill, imm1);
11002}
11003
11004unsigned fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8bf16_ri_Predicate_VectorIndexH(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11005 if (RetVT.SimpleTy != MVT::bf16)
11006 return 0;
11007 return fastEmitInst_ri(AArch64::CPYi16, &AArch64::FPR16RegClass, Op0, Op0IsKill, imm1);
11008}
11009
11010unsigned fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexH(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11011 switch (VT.SimpleTy) {
11012 case MVT::v8i16: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8i16_ri_Predicate_VectorIndexH(RetVT, Op0, Op0IsKill, imm1);
11013 case MVT::v8f16: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8f16_ri_Predicate_VectorIndexH(RetVT, Op0, Op0IsKill, imm1);
11014 case MVT::v8bf16: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8bf16_ri_Predicate_VectorIndexH(RetVT, Op0, Op0IsKill, imm1);
11015 default: return 0;
11016 }
11017}
11018
11019// Top-level FastEmit function.
11020
11021unsigned fastEmit_ri_Predicate_VectorIndexH(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11022 switch (Opcode) {
11023 case AArch64ISD::DUPLANE16: return fastEmit_AArch64ISD_DUPLANE16_ri_Predicate_VectorIndexH(VT, RetVT, Op0, Op0IsKill, imm1);
11024 case ISD::EXTRACT_VECTOR_ELT: return fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexH(VT, RetVT, Op0, Op0IsKill, imm1);
11025 default: return 0;
11026 }
11027}
11028
11029// FastEmit functions for AArch64ISD::DUPLANE8.
11030
11031unsigned fastEmit_AArch64ISD_DUPLANE8_MVT_v16i8_MVT_v8i8_ri_Predicate_VectorIndexB(unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11032 if ((Subtarget->hasNEON())) {
11033 return fastEmitInst_ri(AArch64::DUPv8i8lane, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
11034 }
11035 return 0;
11036}
11037
11038unsigned fastEmit_AArch64ISD_DUPLANE8_MVT_v16i8_MVT_v16i8_ri_Predicate_VectorIndexB(unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11039 if ((Subtarget->hasNEON())) {
11040 return fastEmitInst_ri(AArch64::DUPv16i8lane, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
11041 }
11042 return 0;
11043}
11044
11045unsigned fastEmit_AArch64ISD_DUPLANE8_MVT_v16i8_ri_Predicate_VectorIndexB(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11046switch (RetVT.SimpleTy) {
11047 case MVT::v8i8: return fastEmit_AArch64ISD_DUPLANE8_MVT_v16i8_MVT_v8i8_ri_Predicate_VectorIndexB(Op0, Op0IsKill, imm1);
11048 case MVT::v16i8: return fastEmit_AArch64ISD_DUPLANE8_MVT_v16i8_MVT_v16i8_ri_Predicate_VectorIndexB(Op0, Op0IsKill, imm1);
11049 default: return 0;
11050}
11051}
11052
11053unsigned fastEmit_AArch64ISD_DUPLANE8_ri_Predicate_VectorIndexB(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11054 switch (VT.SimpleTy) {
11055 case MVT::v16i8: return fastEmit_AArch64ISD_DUPLANE8_MVT_v16i8_ri_Predicate_VectorIndexB(RetVT, Op0, Op0IsKill, imm1);
11056 default: return 0;
11057 }
11058}
11059
11060// FastEmit functions for ISD::EXTRACT_VECTOR_ELT.
11061
11062unsigned fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v16i8_ri_Predicate_VectorIndexB(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11063 if (RetVT.SimpleTy != MVT::i32)
11064 return 0;
11065 if ((Subtarget->hasNEON())) {
11066 return fastEmitInst_ri(AArch64::UMOVvi8, &AArch64::GPR32RegClass, Op0, Op0IsKill, imm1);
11067 }
11068 return 0;
11069}
11070
11071unsigned fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexB(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11072 switch (VT.SimpleTy) {
11073 case MVT::v16i8: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v16i8_ri_Predicate_VectorIndexB(RetVT, Op0, Op0IsKill, imm1);
11074 default: return 0;
11075 }
11076}
11077
11078// Top-level FastEmit function.
11079
11080unsigned fastEmit_ri_Predicate_VectorIndexB(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11081 switch (Opcode) {
11082 case AArch64ISD::DUPLANE8: return fastEmit_AArch64ISD_DUPLANE8_ri_Predicate_VectorIndexB(VT, RetVT, Op0, Op0IsKill, imm1);
11083 case ISD::EXTRACT_VECTOR_ELT: return fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexB(VT, RetVT, Op0, Op0IsKill, imm1);
11084 default: return 0;
11085 }
11086}
11087
11088// FastEmit functions for AArch64ISD::SQSHLU_I.
11089
11090unsigned fastEmit_AArch64ISD_SQSHLU_I_MVT_i64_ri_Predicate_vecshiftL64(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11091 if (RetVT.SimpleTy != MVT::i64)
11092 return 0;
11093 if ((Subtarget->hasNEON())) {
11094 return fastEmitInst_ri(AArch64::SQSHLUd, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
11095 }
11096 return 0;
11097}
11098
11099unsigned fastEmit_AArch64ISD_SQSHLU_I_MVT_v1i64_ri_Predicate_vecshiftL64(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11100 if (RetVT.SimpleTy != MVT::v1i64)
11101 return 0;
11102 if ((Subtarget->hasNEON())) {
11103 return fastEmitInst_ri(AArch64::SQSHLUd, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
11104 }
11105 return 0;
11106}
11107
11108unsigned fastEmit_AArch64ISD_SQSHLU_I_MVT_v2i64_ri_Predicate_vecshiftL64(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11109 if (RetVT.SimpleTy != MVT::v2i64)
11110 return 0;
11111 if ((Subtarget->hasNEON())) {
11112 return fastEmitInst_ri(AArch64::SQSHLUv2i64_shift, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
11113 }
11114 return 0;
11115}
11116
11117unsigned fastEmit_AArch64ISD_SQSHLU_I_ri_Predicate_vecshiftL64(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11118 switch (VT.SimpleTy) {
11119 case MVT::i64: return fastEmit_AArch64ISD_SQSHLU_I_MVT_i64_ri_Predicate_vecshiftL64(RetVT, Op0, Op0IsKill, imm1);
11120 case MVT::v1i64: return fastEmit_AArch64ISD_SQSHLU_I_MVT_v1i64_ri_Predicate_vecshiftL64(RetVT, Op0, Op0IsKill, imm1);
11121 case MVT::v2i64: return fastEmit_AArch64ISD_SQSHLU_I_MVT_v2i64_ri_Predicate_vecshiftL64(RetVT, Op0, Op0IsKill, imm1);
11122 default: return 0;
11123 }
11124}
11125
11126// FastEmit functions for AArch64ISD::SQSHL_I.
11127
11128unsigned fastEmit_AArch64ISD_SQSHL_I_MVT_i64_ri_Predicate_vecshiftL64(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11129 if (RetVT.SimpleTy != MVT::i64)
11130 return 0;
11131 if ((Subtarget->hasNEON())) {
11132 return fastEmitInst_ri(AArch64::SQSHLd, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
11133 }
11134 return 0;
11135}
11136
11137unsigned fastEmit_AArch64ISD_SQSHL_I_MVT_v1i64_ri_Predicate_vecshiftL64(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11138 if (RetVT.SimpleTy != MVT::v1i64)
11139 return 0;
11140 if ((Subtarget->hasNEON())) {
11141 return fastEmitInst_ri(AArch64::SQSHLd, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
11142 }
11143 return 0;
11144}
11145
11146unsigned fastEmit_AArch64ISD_SQSHL_I_MVT_v2i64_ri_Predicate_vecshiftL64(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11147 if (RetVT.SimpleTy != MVT::v2i64)
11148 return 0;
11149 if ((Subtarget->hasNEON())) {
11150 return fastEmitInst_ri(AArch64::SQSHLv2i64_shift, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
11151 }
11152 return 0;
11153}
11154
11155unsigned fastEmit_AArch64ISD_SQSHL_I_ri_Predicate_vecshiftL64(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11156 switch (VT.SimpleTy) {
11157 case MVT::i64: return fastEmit_AArch64ISD_SQSHL_I_MVT_i64_ri_Predicate_vecshiftL64(RetVT, Op0, Op0IsKill, imm1);
11158 case MVT::v1i64: return fastEmit_AArch64ISD_SQSHL_I_MVT_v1i64_ri_Predicate_vecshiftL64(RetVT, Op0, Op0IsKill, imm1);
11159 case MVT::v2i64: return fastEmit_AArch64ISD_SQSHL_I_MVT_v2i64_ri_Predicate_vecshiftL64(RetVT, Op0, Op0IsKill, imm1);
11160 default: return 0;
11161 }
11162}
11163
11164// FastEmit functions for AArch64ISD::UQSHL_I.
11165
11166unsigned fastEmit_AArch64ISD_UQSHL_I_MVT_i64_ri_Predicate_vecshiftL64(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11167 if (RetVT.SimpleTy != MVT::i64)
11168 return 0;
11169 if ((Subtarget->hasNEON())) {
11170 return fastEmitInst_ri(AArch64::UQSHLd, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
11171 }
11172 return 0;
11173}
11174
11175unsigned fastEmit_AArch64ISD_UQSHL_I_MVT_v1i64_ri_Predicate_vecshiftL64(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11176 if (RetVT.SimpleTy != MVT::v1i64)
11177 return 0;
11178 if ((Subtarget->hasNEON())) {
11179 return fastEmitInst_ri(AArch64::UQSHLd, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
11180 }
11181 return 0;
11182}
11183
11184unsigned fastEmit_AArch64ISD_UQSHL_I_MVT_v2i64_ri_Predicate_vecshiftL64(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11185 if (RetVT.SimpleTy != MVT::v2i64)
11186 return 0;
11187 if ((Subtarget->hasNEON())) {
11188 return fastEmitInst_ri(AArch64::UQSHLv2i64_shift, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
11189 }
11190 return 0;
11191}
11192
11193unsigned fastEmit_AArch64ISD_UQSHL_I_ri_Predicate_vecshiftL64(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11194 switch (VT.SimpleTy) {
11195 case MVT::i64: return fastEmit_AArch64ISD_UQSHL_I_MVT_i64_ri_Predicate_vecshiftL64(RetVT, Op0, Op0IsKill, imm1);
11196 case MVT::v1i64: return fastEmit_AArch64ISD_UQSHL_I_MVT_v1i64_ri_Predicate_vecshiftL64(RetVT, Op0, Op0IsKill, imm1);
11197 case MVT::v2i64: return fastEmit_AArch64ISD_UQSHL_I_MVT_v2i64_ri_Predicate_vecshiftL64(RetVT, Op0, Op0IsKill, imm1);
11198 default: return 0;
11199 }
11200}
11201
11202// FastEmit functions for AArch64ISD::VSHL.
11203
11204unsigned fastEmit_AArch64ISD_VSHL_MVT_v1i64_ri_Predicate_vecshiftL64(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11205 if (RetVT.SimpleTy != MVT::v1i64)
11206 return 0;
11207 if ((Subtarget->hasNEON())) {
11208 return fastEmitInst_ri(AArch64::SHLd, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
11209 }
11210 return 0;
11211}
11212
11213unsigned fastEmit_AArch64ISD_VSHL_MVT_v2i64_ri_Predicate_vecshiftL64(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11214 if (RetVT.SimpleTy != MVT::v2i64)
11215 return 0;
11216 if ((Subtarget->hasNEON())) {
11217 return fastEmitInst_ri(AArch64::SHLv2i64_shift, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
11218 }
11219 return 0;
11220}
11221
11222unsigned fastEmit_AArch64ISD_VSHL_ri_Predicate_vecshiftL64(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11223 switch (VT.SimpleTy) {
11224 case MVT::v1i64: return fastEmit_AArch64ISD_VSHL_MVT_v1i64_ri_Predicate_vecshiftL64(RetVT, Op0, Op0IsKill, imm1);
11225 case MVT::v2i64: return fastEmit_AArch64ISD_VSHL_MVT_v2i64_ri_Predicate_vecshiftL64(RetVT, Op0, Op0IsKill, imm1);
11226 default: return 0;
11227 }
11228}
11229
11230// Top-level FastEmit function.
11231
11232unsigned fastEmit_ri_Predicate_vecshiftL64(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11233 switch (Opcode) {
11234 case AArch64ISD::SQSHLU_I: return fastEmit_AArch64ISD_SQSHLU_I_ri_Predicate_vecshiftL64(VT, RetVT, Op0, Op0IsKill, imm1);
11235 case AArch64ISD::SQSHL_I: return fastEmit_AArch64ISD_SQSHL_I_ri_Predicate_vecshiftL64(VT, RetVT, Op0, Op0IsKill, imm1);
11236 case AArch64ISD::UQSHL_I: return fastEmit_AArch64ISD_UQSHL_I_ri_Predicate_vecshiftL64(VT, RetVT, Op0, Op0IsKill, imm1);
11237 case AArch64ISD::VSHL: return fastEmit_AArch64ISD_VSHL_ri_Predicate_vecshiftL64(VT, RetVT, Op0, Op0IsKill, imm1);
11238 default: return 0;
11239 }
11240}
11241
11242// FastEmit functions for AArch64ISD::SQSHLU_I.
11243
11244unsigned fastEmit_AArch64ISD_SQSHLU_I_MVT_i32_ri_Predicate_vecshiftL32(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11245 if (RetVT.SimpleTy != MVT::i32)
11246 return 0;
11247 if ((Subtarget->hasNEON())) {
11248 return fastEmitInst_ri(AArch64::SQSHLUs, &AArch64::FPR32RegClass, Op0, Op0IsKill, imm1);
11249 }
11250 return 0;
11251}
11252
11253unsigned fastEmit_AArch64ISD_SQSHLU_I_MVT_v2i32_ri_Predicate_vecshiftL32(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11254 if (RetVT.SimpleTy != MVT::v2i32)
11255 return 0;
11256 if ((Subtarget->hasNEON())) {
11257 return fastEmitInst_ri(AArch64::SQSHLUv2i32_shift, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
11258 }
11259 return 0;
11260}
11261
11262unsigned fastEmit_AArch64ISD_SQSHLU_I_MVT_v4i32_ri_Predicate_vecshiftL32(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11263 if (RetVT.SimpleTy != MVT::v4i32)
11264 return 0;
11265 if ((Subtarget->hasNEON())) {
11266 return fastEmitInst_ri(AArch64::SQSHLUv4i32_shift, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
11267 }
11268 return 0;
11269}
11270
11271unsigned fastEmit_AArch64ISD_SQSHLU_I_ri_Predicate_vecshiftL32(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11272 switch (VT.SimpleTy) {
11273 case MVT::i32: return fastEmit_AArch64ISD_SQSHLU_I_MVT_i32_ri_Predicate_vecshiftL32(RetVT, Op0, Op0IsKill, imm1);
11274 case MVT::v2i32: return fastEmit_AArch64ISD_SQSHLU_I_MVT_v2i32_ri_Predicate_vecshiftL32(RetVT, Op0, Op0IsKill, imm1);
11275 case MVT::v4i32: return fastEmit_AArch64ISD_SQSHLU_I_MVT_v4i32_ri_Predicate_vecshiftL32(RetVT, Op0, Op0IsKill, imm1);
11276 default: return 0;
11277 }
11278}
11279
11280// FastEmit functions for AArch64ISD::SQSHL_I.
11281
11282unsigned fastEmit_AArch64ISD_SQSHL_I_MVT_i32_ri_Predicate_vecshiftL32(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11283 if (RetVT.SimpleTy != MVT::i32)
11284 return 0;
11285 if ((Subtarget->hasNEON())) {
11286 return fastEmitInst_ri(AArch64::SQSHLs, &AArch64::FPR32RegClass, Op0, Op0IsKill, imm1);
11287 }
11288 return 0;
11289}
11290
11291unsigned fastEmit_AArch64ISD_SQSHL_I_MVT_v2i32_ri_Predicate_vecshiftL32(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11292 if (RetVT.SimpleTy != MVT::v2i32)
11293 return 0;
11294 if ((Subtarget->hasNEON())) {
11295 return fastEmitInst_ri(AArch64::SQSHLv2i32_shift, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
11296 }
11297 return 0;
11298}
11299
11300unsigned fastEmit_AArch64ISD_SQSHL_I_MVT_v4i32_ri_Predicate_vecshiftL32(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11301 if (RetVT.SimpleTy != MVT::v4i32)
11302 return 0;
11303 if ((Subtarget->hasNEON())) {
11304 return fastEmitInst_ri(AArch64::SQSHLv4i32_shift, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
11305 }
11306 return 0;
11307}
11308
11309unsigned fastEmit_AArch64ISD_SQSHL_I_ri_Predicate_vecshiftL32(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11310 switch (VT.SimpleTy) {
11311 case MVT::i32: return fastEmit_AArch64ISD_SQSHL_I_MVT_i32_ri_Predicate_vecshiftL32(RetVT, Op0, Op0IsKill, imm1);
11312 case MVT::v2i32: return fastEmit_AArch64ISD_SQSHL_I_MVT_v2i32_ri_Predicate_vecshiftL32(RetVT, Op0, Op0IsKill, imm1);
11313 case MVT::v4i32: return fastEmit_AArch64ISD_SQSHL_I_MVT_v4i32_ri_Predicate_vecshiftL32(RetVT, Op0, Op0IsKill, imm1);
11314 default: return 0;
11315 }
11316}
11317
11318// FastEmit functions for AArch64ISD::UQSHL_I.
11319
11320unsigned fastEmit_AArch64ISD_UQSHL_I_MVT_i32_ri_Predicate_vecshiftL32(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11321 if (RetVT.SimpleTy != MVT::i32)
11322 return 0;
11323 if ((Subtarget->hasNEON())) {
11324 return fastEmitInst_ri(AArch64::UQSHLs, &AArch64::FPR32RegClass, Op0, Op0IsKill, imm1);
11325 }
11326 return 0;
11327}
11328
11329unsigned fastEmit_AArch64ISD_UQSHL_I_MVT_v2i32_ri_Predicate_vecshiftL32(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11330 if (RetVT.SimpleTy != MVT::v2i32)
11331 return 0;
11332 if ((Subtarget->hasNEON())) {
11333 return fastEmitInst_ri(AArch64::UQSHLv2i32_shift, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
11334 }
11335 return 0;
11336}
11337
11338unsigned fastEmit_AArch64ISD_UQSHL_I_MVT_v4i32_ri_Predicate_vecshiftL32(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11339 if (RetVT.SimpleTy != MVT::v4i32)
11340 return 0;
11341 if ((Subtarget->hasNEON())) {
11342 return fastEmitInst_ri(AArch64::UQSHLv4i32_shift, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
11343 }
11344 return 0;
11345}
11346
11347unsigned fastEmit_AArch64ISD_UQSHL_I_ri_Predicate_vecshiftL32(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11348 switch (VT.SimpleTy) {
11349 case MVT::i32: return fastEmit_AArch64ISD_UQSHL_I_MVT_i32_ri_Predicate_vecshiftL32(RetVT, Op0, Op0IsKill, imm1);
11350 case MVT::v2i32: return fastEmit_AArch64ISD_UQSHL_I_MVT_v2i32_ri_Predicate_vecshiftL32(RetVT, Op0, Op0IsKill, imm1);
11351 case MVT::v4i32: return fastEmit_AArch64ISD_UQSHL_I_MVT_v4i32_ri_Predicate_vecshiftL32(RetVT, Op0, Op0IsKill, imm1);
11352 default: return 0;
11353 }
11354}
11355
11356// FastEmit functions for AArch64ISD::VSHL.
11357
11358unsigned fastEmit_AArch64ISD_VSHL_MVT_v2i32_ri_Predicate_vecshiftL32(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11359 if (RetVT.SimpleTy != MVT::v2i32)
11360 return 0;
11361 if ((Subtarget->hasNEON())) {
11362 return fastEmitInst_ri(AArch64::SHLv2i32_shift, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
11363 }
11364 return 0;
11365}
11366
11367unsigned fastEmit_AArch64ISD_VSHL_MVT_v4i32_ri_Predicate_vecshiftL32(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11368 if (RetVT.SimpleTy != MVT::v4i32)
11369 return 0;
11370 if ((Subtarget->hasNEON())) {
11371 return fastEmitInst_ri(AArch64::SHLv4i32_shift, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
11372 }
11373 return 0;
11374}
11375
11376unsigned fastEmit_AArch64ISD_VSHL_ri_Predicate_vecshiftL32(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11377 switch (VT.SimpleTy) {
11378 case MVT::v2i32: return fastEmit_AArch64ISD_VSHL_MVT_v2i32_ri_Predicate_vecshiftL32(RetVT, Op0, Op0IsKill, imm1);
11379 case MVT::v4i32: return fastEmit_AArch64ISD_VSHL_MVT_v4i32_ri_Predicate_vecshiftL32(RetVT, Op0, Op0IsKill, imm1);
11380 default: return 0;
11381 }
11382}
11383
11384// Top-level FastEmit function.
11385
11386unsigned fastEmit_ri_Predicate_vecshiftL32(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11387 switch (Opcode) {
11388 case AArch64ISD::SQSHLU_I: return fastEmit_AArch64ISD_SQSHLU_I_ri_Predicate_vecshiftL32(VT, RetVT, Op0, Op0IsKill, imm1);
11389 case AArch64ISD::SQSHL_I: return fastEmit_AArch64ISD_SQSHL_I_ri_Predicate_vecshiftL32(VT, RetVT, Op0, Op0IsKill, imm1);
11390 case AArch64ISD::UQSHL_I: return fastEmit_AArch64ISD_UQSHL_I_ri_Predicate_vecshiftL32(VT, RetVT, Op0, Op0IsKill, imm1);
11391 case AArch64ISD::VSHL: return fastEmit_AArch64ISD_VSHL_ri_Predicate_vecshiftL32(VT, RetVT, Op0, Op0IsKill, imm1);
11392 default: return 0;
11393 }
11394}
11395
11396// FastEmit functions for AArch64ISD::SRSHR_I.
11397
11398unsigned fastEmit_AArch64ISD_SRSHR_I_MVT_i64_ri_Predicate_vecshiftR64(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11399 if (RetVT.SimpleTy != MVT::i64)
11400 return 0;
11401 if ((Subtarget->hasNEON())) {
11402 return fastEmitInst_ri(AArch64::SRSHRd, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
11403 }
11404 return 0;
11405}
11406
11407unsigned fastEmit_AArch64ISD_SRSHR_I_MVT_v1i64_ri_Predicate_vecshiftR64(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11408 if (RetVT.SimpleTy != MVT::v1i64)
11409 return 0;
11410 if ((Subtarget->hasNEON())) {
11411 return fastEmitInst_ri(AArch64::SRSHRd, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
11412 }
11413 return 0;
11414}
11415
11416unsigned fastEmit_AArch64ISD_SRSHR_I_MVT_v2i64_ri_Predicate_vecshiftR64(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11417 if (RetVT.SimpleTy != MVT::v2i64)
11418 return 0;
11419 if ((Subtarget->hasNEON())) {
11420 return fastEmitInst_ri(AArch64::SRSHRv2i64_shift, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
11421 }
11422 return 0;
11423}
11424
11425unsigned fastEmit_AArch64ISD_SRSHR_I_ri_Predicate_vecshiftR64(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11426 switch (VT.SimpleTy) {
11427 case MVT::i64: return fastEmit_AArch64ISD_SRSHR_I_MVT_i64_ri_Predicate_vecshiftR64(RetVT, Op0, Op0IsKill, imm1);
11428 case MVT::v1i64: return fastEmit_AArch64ISD_SRSHR_I_MVT_v1i64_ri_Predicate_vecshiftR64(RetVT, Op0, Op0IsKill, imm1);
11429 case MVT::v2i64: return fastEmit_AArch64ISD_SRSHR_I_MVT_v2i64_ri_Predicate_vecshiftR64(RetVT, Op0, Op0IsKill, imm1);
11430 default: return 0;
11431 }
11432}
11433
11434// FastEmit functions for AArch64ISD::URSHR_I.
11435
11436unsigned fastEmit_AArch64ISD_URSHR_I_MVT_i64_ri_Predicate_vecshiftR64(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11437 if (RetVT.SimpleTy != MVT::i64)
11438 return 0;
11439 if ((Subtarget->hasNEON())) {
11440 return fastEmitInst_ri(AArch64::URSHRd, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
11441 }
11442 return 0;
11443}
11444
11445unsigned fastEmit_AArch64ISD_URSHR_I_MVT_v1i64_ri_Predicate_vecshiftR64(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11446 if (RetVT.SimpleTy != MVT::v1i64)
11447 return 0;
11448 if ((Subtarget->hasNEON())) {
11449 return fastEmitInst_ri(AArch64::URSHRd, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
11450 }
11451 return 0;
11452}
11453
11454unsigned fastEmit_AArch64ISD_URSHR_I_MVT_v2i64_ri_Predicate_vecshiftR64(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11455 if (RetVT.SimpleTy != MVT::v2i64)
11456 return 0;
11457 if ((Subtarget->hasNEON())) {
11458 return fastEmitInst_ri(AArch64::URSHRv2i64_shift, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
11459 }
11460 return 0;
11461}
11462
11463unsigned fastEmit_AArch64ISD_URSHR_I_ri_Predicate_vecshiftR64(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11464 switch (VT.SimpleTy) {
11465 case MVT::i64: return fastEmit_AArch64ISD_URSHR_I_MVT_i64_ri_Predicate_vecshiftR64(RetVT, Op0, Op0IsKill, imm1);
11466 case MVT::v1i64: return fastEmit_AArch64ISD_URSHR_I_MVT_v1i64_ri_Predicate_vecshiftR64(RetVT, Op0, Op0IsKill, imm1);
11467 case MVT::v2i64: return fastEmit_AArch64ISD_URSHR_I_MVT_v2i64_ri_Predicate_vecshiftR64(RetVT, Op0, Op0IsKill, imm1);
11468 default: return 0;
11469 }
11470}
11471
11472// FastEmit functions for AArch64ISD::VASHR.
11473
11474unsigned fastEmit_AArch64ISD_VASHR_MVT_i64_ri_Predicate_vecshiftR64(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11475 if (RetVT.SimpleTy != MVT::i64)
11476 return 0;
11477 if ((Subtarget->hasNEON())) {
11478 return fastEmitInst_ri(AArch64::SSHRd, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
11479 }
11480 return 0;
11481}
11482
11483unsigned fastEmit_AArch64ISD_VASHR_MVT_v1i64_ri_Predicate_vecshiftR64(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11484 if (RetVT.SimpleTy != MVT::v1i64)
11485 return 0;
11486 if ((Subtarget->hasNEON())) {
11487 return fastEmitInst_ri(AArch64::SSHRd, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
11488 }
11489 return 0;
11490}
11491
11492unsigned fastEmit_AArch64ISD_VASHR_MVT_v2i64_ri_Predicate_vecshiftR64(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11493 if (RetVT.SimpleTy != MVT::v2i64)
11494 return 0;
11495 if ((Subtarget->hasNEON())) {
11496 return fastEmitInst_ri(AArch64::SSHRv2i64_shift, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
11497 }
11498 return 0;
11499}
11500
11501unsigned fastEmit_AArch64ISD_VASHR_ri_Predicate_vecshiftR64(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11502 switch (VT.SimpleTy) {
11503 case MVT::i64: return fastEmit_AArch64ISD_VASHR_MVT_i64_ri_Predicate_vecshiftR64(RetVT, Op0, Op0IsKill, imm1);
11504 case MVT::v1i64: return fastEmit_AArch64ISD_VASHR_MVT_v1i64_ri_Predicate_vecshiftR64(RetVT, Op0, Op0IsKill, imm1);
11505 case MVT::v2i64: return fastEmit_AArch64ISD_VASHR_MVT_v2i64_ri_Predicate_vecshiftR64(RetVT, Op0, Op0IsKill, imm1);
11506 default: return 0;
11507 }
11508}
11509
11510// FastEmit functions for AArch64ISD::VLSHR.
11511
11512unsigned fastEmit_AArch64ISD_VLSHR_MVT_i64_ri_Predicate_vecshiftR64(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11513 if (RetVT.SimpleTy != MVT::i64)
11514 return 0;
11515 if ((Subtarget->hasNEON())) {
11516 return fastEmitInst_ri(AArch64::USHRd, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
11517 }
11518 return 0;
11519}
11520
11521unsigned fastEmit_AArch64ISD_VLSHR_MVT_v1i64_ri_Predicate_vecshiftR64(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11522 if (RetVT.SimpleTy != MVT::v1i64)
11523 return 0;
11524 if ((Subtarget->hasNEON())) {
11525 return fastEmitInst_ri(AArch64::USHRd, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
11526 }
11527 return 0;
11528}
11529
11530unsigned fastEmit_AArch64ISD_VLSHR_MVT_v2i64_ri_Predicate_vecshiftR64(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11531 if (RetVT.SimpleTy != MVT::v2i64)
11532 return 0;
11533 if ((Subtarget->hasNEON())) {
11534 return fastEmitInst_ri(AArch64::USHRv2i64_shift, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
11535 }
11536 return 0;
11537}
11538
11539unsigned fastEmit_AArch64ISD_VLSHR_ri_Predicate_vecshiftR64(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11540 switch (VT.SimpleTy) {
11541 case MVT::i64: return fastEmit_AArch64ISD_VLSHR_MVT_i64_ri_Predicate_vecshiftR64(RetVT, Op0, Op0IsKill, imm1);
11542 case MVT::v1i64: return fastEmit_AArch64ISD_VLSHR_MVT_v1i64_ri_Predicate_vecshiftR64(RetVT, Op0, Op0IsKill, imm1);
11543 case MVT::v2i64: return fastEmit_AArch64ISD_VLSHR_MVT_v2i64_ri_Predicate_vecshiftR64(RetVT, Op0, Op0IsKill, imm1);
11544 default: return 0;
11545 }
11546}
11547
11548// Top-level FastEmit function.
11549
11550unsigned fastEmit_ri_Predicate_vecshiftR64(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11551 switch (Opcode) {
11552 case AArch64ISD::SRSHR_I: return fastEmit_AArch64ISD_SRSHR_I_ri_Predicate_vecshiftR64(VT, RetVT, Op0, Op0IsKill, imm1);
11553 case AArch64ISD::URSHR_I: return fastEmit_AArch64ISD_URSHR_I_ri_Predicate_vecshiftR64(VT, RetVT, Op0, Op0IsKill, imm1);
11554 case AArch64ISD::VASHR: return fastEmit_AArch64ISD_VASHR_ri_Predicate_vecshiftR64(VT, RetVT, Op0, Op0IsKill, imm1);
11555 case AArch64ISD::VLSHR: return fastEmit_AArch64ISD_VLSHR_ri_Predicate_vecshiftR64(VT, RetVT, Op0, Op0IsKill, imm1);
11556 default: return 0;
11557 }
11558}
11559
11560// FastEmit functions for AArch64ISD::SQSHLU_I.
11561
11562unsigned fastEmit_AArch64ISD_SQSHLU_I_MVT_v8i8_ri_Predicate_vecshiftL8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11563 if (RetVT.SimpleTy != MVT::v8i8)
11564 return 0;
11565 if ((Subtarget->hasNEON())) {
11566 return fastEmitInst_ri(AArch64::SQSHLUv8i8_shift, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
11567 }
11568 return 0;
11569}
11570
11571unsigned fastEmit_AArch64ISD_SQSHLU_I_MVT_v16i8_ri_Predicate_vecshiftL8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11572 if (RetVT.SimpleTy != MVT::v16i8)
11573 return 0;
11574 if ((Subtarget->hasNEON())) {
11575 return fastEmitInst_ri(AArch64::SQSHLUv16i8_shift, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
11576 }
11577 return 0;
11578}
11579
11580unsigned fastEmit_AArch64ISD_SQSHLU_I_ri_Predicate_vecshiftL8(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11581 switch (VT.SimpleTy) {
11582 case MVT::v8i8: return fastEmit_AArch64ISD_SQSHLU_I_MVT_v8i8_ri_Predicate_vecshiftL8(RetVT, Op0, Op0IsKill, imm1);
11583 case MVT::v16i8: return fastEmit_AArch64ISD_SQSHLU_I_MVT_v16i8_ri_Predicate_vecshiftL8(RetVT, Op0, Op0IsKill, imm1);
11584 default: return 0;
11585 }
11586}
11587
11588// FastEmit functions for AArch64ISD::SQSHL_I.
11589
11590unsigned fastEmit_AArch64ISD_SQSHL_I_MVT_v8i8_ri_Predicate_vecshiftL8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11591 if (RetVT.SimpleTy != MVT::v8i8)
11592 return 0;
11593 if ((Subtarget->hasNEON())) {
11594 return fastEmitInst_ri(AArch64::SQSHLv8i8_shift, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
11595 }
11596 return 0;
11597}
11598
11599unsigned fastEmit_AArch64ISD_SQSHL_I_MVT_v16i8_ri_Predicate_vecshiftL8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11600 if (RetVT.SimpleTy != MVT::v16i8)
11601 return 0;
11602 if ((Subtarget->hasNEON())) {
11603 return fastEmitInst_ri(AArch64::SQSHLv16i8_shift, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
11604 }
11605 return 0;
11606}
11607
11608unsigned fastEmit_AArch64ISD_SQSHL_I_ri_Predicate_vecshiftL8(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11609 switch (VT.SimpleTy) {
11610 case MVT::v8i8: return fastEmit_AArch64ISD_SQSHL_I_MVT_v8i8_ri_Predicate_vecshiftL8(RetVT, Op0, Op0IsKill, imm1);
11611 case MVT::v16i8: return fastEmit_AArch64ISD_SQSHL_I_MVT_v16i8_ri_Predicate_vecshiftL8(RetVT, Op0, Op0IsKill, imm1);
11612 default: return 0;
11613 }
11614}
11615
11616// FastEmit functions for AArch64ISD::UQSHL_I.
11617
11618unsigned fastEmit_AArch64ISD_UQSHL_I_MVT_v8i8_ri_Predicate_vecshiftL8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11619 if (RetVT.SimpleTy != MVT::v8i8)
11620 return 0;
11621 if ((Subtarget->hasNEON())) {
11622 return fastEmitInst_ri(AArch64::UQSHLv8i8_shift, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
11623 }
11624 return 0;
11625}
11626
11627unsigned fastEmit_AArch64ISD_UQSHL_I_MVT_v16i8_ri_Predicate_vecshiftL8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11628 if (RetVT.SimpleTy != MVT::v16i8)
11629 return 0;
11630 if ((Subtarget->hasNEON())) {
11631 return fastEmitInst_ri(AArch64::UQSHLv16i8_shift, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
11632 }
11633 return 0;
11634}
11635
11636unsigned fastEmit_AArch64ISD_UQSHL_I_ri_Predicate_vecshiftL8(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11637 switch (VT.SimpleTy) {
11638 case MVT::v8i8: return fastEmit_AArch64ISD_UQSHL_I_MVT_v8i8_ri_Predicate_vecshiftL8(RetVT, Op0, Op0IsKill, imm1);
11639 case MVT::v16i8: return fastEmit_AArch64ISD_UQSHL_I_MVT_v16i8_ri_Predicate_vecshiftL8(RetVT, Op0, Op0IsKill, imm1);
11640 default: return 0;
11641 }
11642}
11643
11644// FastEmit functions for AArch64ISD::VSHL.
11645
11646unsigned fastEmit_AArch64ISD_VSHL_MVT_v8i8_ri_Predicate_vecshiftL8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11647 if (RetVT.SimpleTy != MVT::v8i8)
11648 return 0;
11649 if ((Subtarget->hasNEON())) {
11650 return fastEmitInst_ri(AArch64::SHLv8i8_shift, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
11651 }
11652 return 0;
11653}
11654
11655unsigned fastEmit_AArch64ISD_VSHL_MVT_v16i8_ri_Predicate_vecshiftL8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11656 if (RetVT.SimpleTy != MVT::v16i8)
11657 return 0;
11658 if ((Subtarget->hasNEON())) {
11659 return fastEmitInst_ri(AArch64::SHLv16i8_shift, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
11660 }
11661 return 0;
11662}
11663
11664unsigned fastEmit_AArch64ISD_VSHL_ri_Predicate_vecshiftL8(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11665 switch (VT.SimpleTy) {
11666 case MVT::v8i8: return fastEmit_AArch64ISD_VSHL_MVT_v8i8_ri_Predicate_vecshiftL8(RetVT, Op0, Op0IsKill, imm1);
11667 case MVT::v16i8: return fastEmit_AArch64ISD_VSHL_MVT_v16i8_ri_Predicate_vecshiftL8(RetVT, Op0, Op0IsKill, imm1);
11668 default: return 0;
11669 }
11670}
11671
11672// Top-level FastEmit function.
11673
11674unsigned fastEmit_ri_Predicate_vecshiftL8(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11675 switch (Opcode) {
11676 case AArch64ISD::SQSHLU_I: return fastEmit_AArch64ISD_SQSHLU_I_ri_Predicate_vecshiftL8(VT, RetVT, Op0, Op0IsKill, imm1);
11677 case AArch64ISD::SQSHL_I: return fastEmit_AArch64ISD_SQSHL_I_ri_Predicate_vecshiftL8(VT, RetVT, Op0, Op0IsKill, imm1);
11678 case AArch64ISD::UQSHL_I: return fastEmit_AArch64ISD_UQSHL_I_ri_Predicate_vecshiftL8(VT, RetVT, Op0, Op0IsKill, imm1);
11679 case AArch64ISD::VSHL: return fastEmit_AArch64ISD_VSHL_ri_Predicate_vecshiftL8(VT, RetVT, Op0, Op0IsKill, imm1);
11680 default: return 0;
11681 }
11682}
11683
11684// FastEmit functions for AArch64ISD::SQSHLU_I.
11685
11686unsigned fastEmit_AArch64ISD_SQSHLU_I_MVT_v4i16_ri_Predicate_vecshiftL16(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11687 if (RetVT.SimpleTy != MVT::v4i16)
11688 return 0;
11689 if ((Subtarget->hasNEON())) {
11690 return fastEmitInst_ri(AArch64::SQSHLUv4i16_shift, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
11691 }
11692 return 0;
11693}
11694
11695unsigned fastEmit_AArch64ISD_SQSHLU_I_MVT_v8i16_ri_Predicate_vecshiftL16(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11696 if (RetVT.SimpleTy != MVT::v8i16)
11697 return 0;
11698 if ((Subtarget->hasNEON())) {
11699 return fastEmitInst_ri(AArch64::SQSHLUv8i16_shift, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
11700 }
11701 return 0;
11702}
11703
11704unsigned fastEmit_AArch64ISD_SQSHLU_I_ri_Predicate_vecshiftL16(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11705 switch (VT.SimpleTy) {
11706 case MVT::v4i16: return fastEmit_AArch64ISD_SQSHLU_I_MVT_v4i16_ri_Predicate_vecshiftL16(RetVT, Op0, Op0IsKill, imm1);
11707 case MVT::v8i16: return fastEmit_AArch64ISD_SQSHLU_I_MVT_v8i16_ri_Predicate_vecshiftL16(RetVT, Op0, Op0IsKill, imm1);
11708 default: return 0;
11709 }
11710}
11711
11712// FastEmit functions for AArch64ISD::SQSHL_I.
11713
11714unsigned fastEmit_AArch64ISD_SQSHL_I_MVT_v4i16_ri_Predicate_vecshiftL16(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11715 if (RetVT.SimpleTy != MVT::v4i16)
11716 return 0;
11717 if ((Subtarget->hasNEON())) {
11718 return fastEmitInst_ri(AArch64::SQSHLv4i16_shift, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
11719 }
11720 return 0;
11721}
11722
11723unsigned fastEmit_AArch64ISD_SQSHL_I_MVT_v8i16_ri_Predicate_vecshiftL16(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11724 if (RetVT.SimpleTy != MVT::v8i16)
11725 return 0;
11726 if ((Subtarget->hasNEON())) {
11727 return fastEmitInst_ri(AArch64::SQSHLv8i16_shift, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
11728 }
11729 return 0;
11730}
11731
11732unsigned fastEmit_AArch64ISD_SQSHL_I_ri_Predicate_vecshiftL16(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11733 switch (VT.SimpleTy) {
11734 case MVT::v4i16: return fastEmit_AArch64ISD_SQSHL_I_MVT_v4i16_ri_Predicate_vecshiftL16(RetVT, Op0, Op0IsKill, imm1);
11735 case MVT::v8i16: return fastEmit_AArch64ISD_SQSHL_I_MVT_v8i16_ri_Predicate_vecshiftL16(RetVT, Op0, Op0IsKill, imm1);
11736 default: return 0;
11737 }
11738}
11739
11740// FastEmit functions for AArch64ISD::UQSHL_I.
11741
11742unsigned fastEmit_AArch64ISD_UQSHL_I_MVT_v4i16_ri_Predicate_vecshiftL16(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11743 if (RetVT.SimpleTy != MVT::v4i16)
11744 return 0;
11745 if ((Subtarget->hasNEON())) {
11746 return fastEmitInst_ri(AArch64::UQSHLv4i16_shift, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
11747 }
11748 return 0;
11749}
11750
11751unsigned fastEmit_AArch64ISD_UQSHL_I_MVT_v8i16_ri_Predicate_vecshiftL16(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11752 if (RetVT.SimpleTy != MVT::v8i16)
11753 return 0;
11754 if ((Subtarget->hasNEON())) {
11755 return fastEmitInst_ri(AArch64::UQSHLv8i16_shift, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
11756 }
11757 return 0;
11758}
11759
11760unsigned fastEmit_AArch64ISD_UQSHL_I_ri_Predicate_vecshiftL16(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11761 switch (VT.SimpleTy) {
11762 case MVT::v4i16: return fastEmit_AArch64ISD_UQSHL_I_MVT_v4i16_ri_Predicate_vecshiftL16(RetVT, Op0, Op0IsKill, imm1);
11763 case MVT::v8i16: return fastEmit_AArch64ISD_UQSHL_I_MVT_v8i16_ri_Predicate_vecshiftL16(RetVT, Op0, Op0IsKill, imm1);
11764 default: return 0;
11765 }
11766}
11767
11768// FastEmit functions for AArch64ISD::VSHL.
11769
11770unsigned fastEmit_AArch64ISD_VSHL_MVT_v4i16_ri_Predicate_vecshiftL16(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11771 if (RetVT.SimpleTy != MVT::v4i16)
11772 return 0;
11773 if ((Subtarget->hasNEON())) {
11774 return fastEmitInst_ri(AArch64::SHLv4i16_shift, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
11775 }
11776 return 0;
11777}
11778
11779unsigned fastEmit_AArch64ISD_VSHL_MVT_v8i16_ri_Predicate_vecshiftL16(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11780 if (RetVT.SimpleTy != MVT::v8i16)
11781 return 0;
11782 if ((Subtarget->hasNEON())) {
11783 return fastEmitInst_ri(AArch64::SHLv8i16_shift, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
11784 }
11785 return 0;
11786}
11787
11788unsigned fastEmit_AArch64ISD_VSHL_ri_Predicate_vecshiftL16(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11789 switch (VT.SimpleTy) {
11790 case MVT::v4i16: return fastEmit_AArch64ISD_VSHL_MVT_v4i16_ri_Predicate_vecshiftL16(RetVT, Op0, Op0IsKill, imm1);
11791 case MVT::v8i16: return fastEmit_AArch64ISD_VSHL_MVT_v8i16_ri_Predicate_vecshiftL16(RetVT, Op0, Op0IsKill, imm1);
11792 default: return 0;
11793 }
11794}
11795
11796// Top-level FastEmit function.
11797
11798unsigned fastEmit_ri_Predicate_vecshiftL16(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11799 switch (Opcode) {
11800 case AArch64ISD::SQSHLU_I: return fastEmit_AArch64ISD_SQSHLU_I_ri_Predicate_vecshiftL16(VT, RetVT, Op0, Op0IsKill, imm1);
11801 case AArch64ISD::SQSHL_I: return fastEmit_AArch64ISD_SQSHL_I_ri_Predicate_vecshiftL16(VT, RetVT, Op0, Op0IsKill, imm1);
11802 case AArch64ISD::UQSHL_I: return fastEmit_AArch64ISD_UQSHL_I_ri_Predicate_vecshiftL16(VT, RetVT, Op0, Op0IsKill, imm1);
11803 case AArch64ISD::VSHL: return fastEmit_AArch64ISD_VSHL_ri_Predicate_vecshiftL16(VT, RetVT, Op0, Op0IsKill, imm1);
11804 default: return 0;
11805 }
11806}
11807
11808// FastEmit functions for AArch64ISD::SRSHR_I.
11809
11810unsigned fastEmit_AArch64ISD_SRSHR_I_MVT_v8i8_ri_Predicate_vecshiftR8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11811 if (RetVT.SimpleTy != MVT::v8i8)
11812 return 0;
11813 if ((Subtarget->hasNEON())) {
11814 return fastEmitInst_ri(AArch64::SRSHRv8i8_shift, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
11815 }
11816 return 0;
11817}
11818
11819unsigned fastEmit_AArch64ISD_SRSHR_I_MVT_v16i8_ri_Predicate_vecshiftR8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11820 if (RetVT.SimpleTy != MVT::v16i8)
11821 return 0;
11822 if ((Subtarget->hasNEON())) {
11823 return fastEmitInst_ri(AArch64::SRSHRv16i8_shift, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
11824 }
11825 return 0;
11826}
11827
11828unsigned fastEmit_AArch64ISD_SRSHR_I_ri_Predicate_vecshiftR8(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11829 switch (VT.SimpleTy) {
11830 case MVT::v8i8: return fastEmit_AArch64ISD_SRSHR_I_MVT_v8i8_ri_Predicate_vecshiftR8(RetVT, Op0, Op0IsKill, imm1);
11831 case MVT::v16i8: return fastEmit_AArch64ISD_SRSHR_I_MVT_v16i8_ri_Predicate_vecshiftR8(RetVT, Op0, Op0IsKill, imm1);
11832 default: return 0;
11833 }
11834}
11835
11836// FastEmit functions for AArch64ISD::URSHR_I.
11837
11838unsigned fastEmit_AArch64ISD_URSHR_I_MVT_v8i8_ri_Predicate_vecshiftR8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11839 if (RetVT.SimpleTy != MVT::v8i8)
11840 return 0;
11841 if ((Subtarget->hasNEON())) {
11842 return fastEmitInst_ri(AArch64::URSHRv8i8_shift, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
11843 }
11844 return 0;
11845}
11846
11847unsigned fastEmit_AArch64ISD_URSHR_I_MVT_v16i8_ri_Predicate_vecshiftR8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11848 if (RetVT.SimpleTy != MVT::v16i8)
11849 return 0;
11850 if ((Subtarget->hasNEON())) {
11851 return fastEmitInst_ri(AArch64::URSHRv16i8_shift, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
11852 }
11853 return 0;
11854}
11855
11856unsigned fastEmit_AArch64ISD_URSHR_I_ri_Predicate_vecshiftR8(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11857 switch (VT.SimpleTy) {
11858 case MVT::v8i8: return fastEmit_AArch64ISD_URSHR_I_MVT_v8i8_ri_Predicate_vecshiftR8(RetVT, Op0, Op0IsKill, imm1);
11859 case MVT::v16i8: return fastEmit_AArch64ISD_URSHR_I_MVT_v16i8_ri_Predicate_vecshiftR8(RetVT, Op0, Op0IsKill, imm1);
11860 default: return 0;
11861 }
11862}
11863
11864// FastEmit functions for AArch64ISD::VASHR.
11865
11866unsigned fastEmit_AArch64ISD_VASHR_MVT_v8i8_ri_Predicate_vecshiftR8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11867 if (RetVT.SimpleTy != MVT::v8i8)
11868 return 0;
11869 if ((Subtarget->hasNEON())) {
11870 return fastEmitInst_ri(AArch64::SSHRv8i8_shift, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
11871 }
11872 return 0;
11873}
11874
11875unsigned fastEmit_AArch64ISD_VASHR_MVT_v16i8_ri_Predicate_vecshiftR8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11876 if (RetVT.SimpleTy != MVT::v16i8)
11877 return 0;
11878 if ((Subtarget->hasNEON())) {
11879 return fastEmitInst_ri(AArch64::SSHRv16i8_shift, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
11880 }
11881 return 0;
11882}
11883
11884unsigned fastEmit_AArch64ISD_VASHR_ri_Predicate_vecshiftR8(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11885 switch (VT.SimpleTy) {
11886 case MVT::v8i8: return fastEmit_AArch64ISD_VASHR_MVT_v8i8_ri_Predicate_vecshiftR8(RetVT, Op0, Op0IsKill, imm1);
11887 case MVT::v16i8: return fastEmit_AArch64ISD_VASHR_MVT_v16i8_ri_Predicate_vecshiftR8(RetVT, Op0, Op0IsKill, imm1);
11888 default: return 0;
11889 }
11890}
11891
11892// FastEmit functions for AArch64ISD::VLSHR.
11893
11894unsigned fastEmit_AArch64ISD_VLSHR_MVT_v8i8_ri_Predicate_vecshiftR8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11895 if (RetVT.SimpleTy != MVT::v8i8)
11896 return 0;
11897 if ((Subtarget->hasNEON())) {
11898 return fastEmitInst_ri(AArch64::USHRv8i8_shift, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
11899 }
11900 return 0;
11901}
11902
11903unsigned fastEmit_AArch64ISD_VLSHR_MVT_v16i8_ri_Predicate_vecshiftR8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11904 if (RetVT.SimpleTy != MVT::v16i8)
11905 return 0;
11906 if ((Subtarget->hasNEON())) {
11907 return fastEmitInst_ri(AArch64::USHRv16i8_shift, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
11908 }
11909 return 0;
11910}
11911
11912unsigned fastEmit_AArch64ISD_VLSHR_ri_Predicate_vecshiftR8(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11913 switch (VT.SimpleTy) {
11914 case MVT::v8i8: return fastEmit_AArch64ISD_VLSHR_MVT_v8i8_ri_Predicate_vecshiftR8(RetVT, Op0, Op0IsKill, imm1);
11915 case MVT::v16i8: return fastEmit_AArch64ISD_VLSHR_MVT_v16i8_ri_Predicate_vecshiftR8(RetVT, Op0, Op0IsKill, imm1);
11916 default: return 0;
11917 }
11918}
11919
11920// Top-level FastEmit function.
11921
11922unsigned fastEmit_ri_Predicate_vecshiftR8(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11923 switch (Opcode) {
11924 case AArch64ISD::SRSHR_I: return fastEmit_AArch64ISD_SRSHR_I_ri_Predicate_vecshiftR8(VT, RetVT, Op0, Op0IsKill, imm1);
11925 case AArch64ISD::URSHR_I: return fastEmit_AArch64ISD_URSHR_I_ri_Predicate_vecshiftR8(VT, RetVT, Op0, Op0IsKill, imm1);
11926 case AArch64ISD::VASHR: return fastEmit_AArch64ISD_VASHR_ri_Predicate_vecshiftR8(VT, RetVT, Op0, Op0IsKill, imm1);
11927 case AArch64ISD::VLSHR: return fastEmit_AArch64ISD_VLSHR_ri_Predicate_vecshiftR8(VT, RetVT, Op0, Op0IsKill, imm1);
11928 default: return 0;
11929 }
11930}
11931
11932// FastEmit functions for AArch64ISD::SRSHR_I.
11933
11934unsigned fastEmit_AArch64ISD_SRSHR_I_MVT_v4i16_ri_Predicate_vecshiftR16(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11935 if (RetVT.SimpleTy != MVT::v4i16)
11936 return 0;
11937 if ((Subtarget->hasNEON())) {
11938 return fastEmitInst_ri(AArch64::SRSHRv4i16_shift, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
11939 }
11940 return 0;
11941}
11942
11943unsigned fastEmit_AArch64ISD_SRSHR_I_MVT_v8i16_ri_Predicate_vecshiftR16(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11944 if (RetVT.SimpleTy != MVT::v8i16)
11945 return 0;
11946 if ((Subtarget->hasNEON())) {
11947 return fastEmitInst_ri(AArch64::SRSHRv8i16_shift, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
11948 }
11949 return 0;
11950}
11951
11952unsigned fastEmit_AArch64ISD_SRSHR_I_ri_Predicate_vecshiftR16(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11953 switch (VT.SimpleTy) {
11954 case MVT::v4i16: return fastEmit_AArch64ISD_SRSHR_I_MVT_v4i16_ri_Predicate_vecshiftR16(RetVT, Op0, Op0IsKill, imm1);
11955 case MVT::v8i16: return fastEmit_AArch64ISD_SRSHR_I_MVT_v8i16_ri_Predicate_vecshiftR16(RetVT, Op0, Op0IsKill, imm1);
11956 default: return 0;
11957 }
11958}
11959
11960// FastEmit functions for AArch64ISD::URSHR_I.
11961
11962unsigned fastEmit_AArch64ISD_URSHR_I_MVT_v4i16_ri_Predicate_vecshiftR16(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11963 if (RetVT.SimpleTy != MVT::v4i16)
11964 return 0;
11965 if ((Subtarget->hasNEON())) {
11966 return fastEmitInst_ri(AArch64::URSHRv4i16_shift, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
11967 }
11968 return 0;
11969}
11970
11971unsigned fastEmit_AArch64ISD_URSHR_I_MVT_v8i16_ri_Predicate_vecshiftR16(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11972 if (RetVT.SimpleTy != MVT::v8i16)
11973 return 0;
11974 if ((Subtarget->hasNEON())) {
11975 return fastEmitInst_ri(AArch64::URSHRv8i16_shift, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
11976 }
11977 return 0;
11978}
11979
11980unsigned fastEmit_AArch64ISD_URSHR_I_ri_Predicate_vecshiftR16(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11981 switch (VT.SimpleTy) {
11982 case MVT::v4i16: return fastEmit_AArch64ISD_URSHR_I_MVT_v4i16_ri_Predicate_vecshiftR16(RetVT, Op0, Op0IsKill, imm1);
11983 case MVT::v8i16: return fastEmit_AArch64ISD_URSHR_I_MVT_v8i16_ri_Predicate_vecshiftR16(RetVT, Op0, Op0IsKill, imm1);
11984 default: return 0;
11985 }
11986}
11987
11988// FastEmit functions for AArch64ISD::VASHR.
11989
11990unsigned fastEmit_AArch64ISD_VASHR_MVT_v4i16_ri_Predicate_vecshiftR16(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
11991 if (RetVT.SimpleTy != MVT::v4i16)
11992 return 0;
11993 if ((Subtarget->hasNEON())) {
11994 return fastEmitInst_ri(AArch64::SSHRv4i16_shift, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
11995 }
11996 return 0;
11997}
11998
11999unsigned fastEmit_AArch64ISD_VASHR_MVT_v8i16_ri_Predicate_vecshiftR16(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
12000 if (RetVT.SimpleTy != MVT::v8i16)
12001 return 0;
12002 if ((Subtarget->hasNEON())) {
12003 return fastEmitInst_ri(AArch64::SSHRv8i16_shift, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
12004 }
12005 return 0;
12006}
12007
12008unsigned fastEmit_AArch64ISD_VASHR_ri_Predicate_vecshiftR16(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
12009 switch (VT.SimpleTy) {
12010 case MVT::v4i16: return fastEmit_AArch64ISD_VASHR_MVT_v4i16_ri_Predicate_vecshiftR16(RetVT, Op0, Op0IsKill, imm1);
12011 case MVT::v8i16: return fastEmit_AArch64ISD_VASHR_MVT_v8i16_ri_Predicate_vecshiftR16(RetVT, Op0, Op0IsKill, imm1);
12012 default: return 0;
12013 }
12014}
12015
12016// FastEmit functions for AArch64ISD::VLSHR.
12017
12018unsigned fastEmit_AArch64ISD_VLSHR_MVT_v4i16_ri_Predicate_vecshiftR16(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
12019 if (RetVT.SimpleTy != MVT::v4i16)
12020 return 0;
12021 if ((Subtarget->hasNEON())) {
12022 return fastEmitInst_ri(AArch64::USHRv4i16_shift, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
12023 }
12024 return 0;
12025}
12026
12027unsigned fastEmit_AArch64ISD_VLSHR_MVT_v8i16_ri_Predicate_vecshiftR16(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
12028 if (RetVT.SimpleTy != MVT::v8i16)
12029 return 0;
12030 if ((Subtarget->hasNEON())) {
12031 return fastEmitInst_ri(AArch64::USHRv8i16_shift, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
12032 }
12033 return 0;
12034}
12035
12036unsigned fastEmit_AArch64ISD_VLSHR_ri_Predicate_vecshiftR16(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
12037 switch (VT.SimpleTy) {
12038 case MVT::v4i16: return fastEmit_AArch64ISD_VLSHR_MVT_v4i16_ri_Predicate_vecshiftR16(RetVT, Op0, Op0IsKill, imm1);
12039 case MVT::v8i16: return fastEmit_AArch64ISD_VLSHR_MVT_v8i16_ri_Predicate_vecshiftR16(RetVT, Op0, Op0IsKill, imm1);
12040 default: return 0;
12041 }
12042}
12043
12044// Top-level FastEmit function.
12045
12046unsigned fastEmit_ri_Predicate_vecshiftR16(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
12047 switch (Opcode) {
12048 case AArch64ISD::SRSHR_I: return fastEmit_AArch64ISD_SRSHR_I_ri_Predicate_vecshiftR16(VT, RetVT, Op0, Op0IsKill, imm1);
12049 case AArch64ISD::URSHR_I: return fastEmit_AArch64ISD_URSHR_I_ri_Predicate_vecshiftR16(VT, RetVT, Op0, Op0IsKill, imm1);
12050 case AArch64ISD::VASHR: return fastEmit_AArch64ISD_VASHR_ri_Predicate_vecshiftR16(VT, RetVT, Op0, Op0IsKill, imm1);
12051 case AArch64ISD::VLSHR: return fastEmit_AArch64ISD_VLSHR_ri_Predicate_vecshiftR16(VT, RetVT, Op0, Op0IsKill, imm1);
12052 default: return 0;
12053 }
12054}
12055
12056// FastEmit functions for AArch64ISD::SRSHR_I.
12057
12058unsigned fastEmit_AArch64ISD_SRSHR_I_MVT_v2i32_ri_Predicate_vecshiftR32(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
12059 if (RetVT.SimpleTy != MVT::v2i32)
12060 return 0;
12061 if ((Subtarget->hasNEON())) {
12062 return fastEmitInst_ri(AArch64::SRSHRv2i32_shift, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
12063 }
12064 return 0;
12065}
12066
12067unsigned fastEmit_AArch64ISD_SRSHR_I_MVT_v4i32_ri_Predicate_vecshiftR32(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
12068 if (RetVT.SimpleTy != MVT::v4i32)
12069 return 0;
12070 if ((Subtarget->hasNEON())) {
12071 return fastEmitInst_ri(AArch64::SRSHRv4i32_shift, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
12072 }
12073 return 0;
12074}
12075
12076unsigned fastEmit_AArch64ISD_SRSHR_I_ri_Predicate_vecshiftR32(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
12077 switch (VT.SimpleTy) {
12078 case MVT::v2i32: return fastEmit_AArch64ISD_SRSHR_I_MVT_v2i32_ri_Predicate_vecshiftR32(RetVT, Op0, Op0IsKill, imm1);
12079 case MVT::v4i32: return fastEmit_AArch64ISD_SRSHR_I_MVT_v4i32_ri_Predicate_vecshiftR32(RetVT, Op0, Op0IsKill, imm1);
12080 default: return 0;
12081 }
12082}
12083
12084// FastEmit functions for AArch64ISD::URSHR_I.
12085
12086unsigned fastEmit_AArch64ISD_URSHR_I_MVT_v2i32_ri_Predicate_vecshiftR32(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
12087 if (RetVT.SimpleTy != MVT::v2i32)
12088 return 0;
12089 if ((Subtarget->hasNEON())) {
12090 return fastEmitInst_ri(AArch64::URSHRv2i32_shift, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
12091 }
12092 return 0;
12093}
12094
12095unsigned fastEmit_AArch64ISD_URSHR_I_MVT_v4i32_ri_Predicate_vecshiftR32(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
12096 if (RetVT.SimpleTy != MVT::v4i32)
12097 return 0;
12098 if ((Subtarget->hasNEON())) {
12099 return fastEmitInst_ri(AArch64::URSHRv4i32_shift, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
12100 }
12101 return 0;
12102}
12103
12104unsigned fastEmit_AArch64ISD_URSHR_I_ri_Predicate_vecshiftR32(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
12105 switch (VT.SimpleTy) {
12106 case MVT::v2i32: return fastEmit_AArch64ISD_URSHR_I_MVT_v2i32_ri_Predicate_vecshiftR32(RetVT, Op0, Op0IsKill, imm1);
12107 case MVT::v4i32: return fastEmit_AArch64ISD_URSHR_I_MVT_v4i32_ri_Predicate_vecshiftR32(RetVT, Op0, Op0IsKill, imm1);
12108 default: return 0;
12109 }
12110}
12111
12112// FastEmit functions for AArch64ISD::VASHR.
12113
12114unsigned fastEmit_AArch64ISD_VASHR_MVT_v2i32_ri_Predicate_vecshiftR32(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
12115 if (RetVT.SimpleTy != MVT::v2i32)
12116 return 0;
12117 if ((Subtarget->hasNEON())) {
12118 return fastEmitInst_ri(AArch64::SSHRv2i32_shift, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
12119 }
12120 return 0;
12121}
12122
12123unsigned fastEmit_AArch64ISD_VASHR_MVT_v4i32_ri_Predicate_vecshiftR32(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
12124 if (RetVT.SimpleTy != MVT::v4i32)
12125 return 0;
12126 if ((Subtarget->hasNEON())) {
12127 return fastEmitInst_ri(AArch64::SSHRv4i32_shift, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
12128 }
12129 return 0;
12130}
12131
12132unsigned fastEmit_AArch64ISD_VASHR_ri_Predicate_vecshiftR32(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
12133 switch (VT.SimpleTy) {
12134 case MVT::v2i32: return fastEmit_AArch64ISD_VASHR_MVT_v2i32_ri_Predicate_vecshiftR32(RetVT, Op0, Op0IsKill, imm1);
12135 case MVT::v4i32: return fastEmit_AArch64ISD_VASHR_MVT_v4i32_ri_Predicate_vecshiftR32(RetVT, Op0, Op0IsKill, imm1);
12136 default: return 0;
12137 }
12138}
12139
12140// FastEmit functions for AArch64ISD::VLSHR.
12141
12142unsigned fastEmit_AArch64ISD_VLSHR_MVT_v2i32_ri_Predicate_vecshiftR32(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
12143 if (RetVT.SimpleTy != MVT::v2i32)
12144 return 0;
12145 if ((Subtarget->hasNEON())) {
12146 return fastEmitInst_ri(AArch64::USHRv2i32_shift, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
12147 }
12148 return 0;
12149}
12150
12151unsigned fastEmit_AArch64ISD_VLSHR_MVT_v4i32_ri_Predicate_vecshiftR32(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
12152 if (RetVT.SimpleTy != MVT::v4i32)
12153 return 0;
12154 if ((Subtarget->hasNEON())) {
12155 return fastEmitInst_ri(AArch64::USHRv4i32_shift, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
12156 }
12157 return 0;
12158}
12159
12160unsigned fastEmit_AArch64ISD_VLSHR_ri_Predicate_vecshiftR32(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
12161 switch (VT.SimpleTy) {
12162 case MVT::v2i32: return fastEmit_AArch64ISD_VLSHR_MVT_v2i32_ri_Predicate_vecshiftR32(RetVT, Op0, Op0IsKill, imm1);
12163 case MVT::v4i32: return fastEmit_AArch64ISD_VLSHR_MVT_v4i32_ri_Predicate_vecshiftR32(RetVT, Op0, Op0IsKill, imm1);
12164 default: return 0;
12165 }
12166}
12167
12168// Top-level FastEmit function.
12169
12170unsigned fastEmit_ri_Predicate_vecshiftR32(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
12171 switch (Opcode) {
12172 case AArch64ISD::SRSHR_I: return fastEmit_AArch64ISD_SRSHR_I_ri_Predicate_vecshiftR32(VT, RetVT, Op0, Op0IsKill, imm1);
12173 case AArch64ISD::URSHR_I: return fastEmit_AArch64ISD_URSHR_I_ri_Predicate_vecshiftR32(VT, RetVT, Op0, Op0IsKill, imm1);
12174 case AArch64ISD::VASHR: return fastEmit_AArch64ISD_VASHR_ri_Predicate_vecshiftR32(VT, RetVT, Op0, Op0IsKill, imm1);
12175 case AArch64ISD::VLSHR: return fastEmit_AArch64ISD_VLSHR_ri_Predicate_vecshiftR32(VT, RetVT, Op0, Op0IsKill, imm1);
12176 default: return 0;
12177 }
12178}
12179
12180// FastEmit functions for AArch64ISD::INDEX_VECTOR.
12181
12182unsigned fastEmit_AArch64ISD_INDEX_VECTOR_MVT_i32_ri_Predicate_simm5_8b(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
12183 if (RetVT.SimpleTy != MVT::nxv16i8)
12184 return 0;
12185 if ((Subtarget->hasSVE())) {
12186 return fastEmitInst_ri(AArch64::INDEX_RI_B, &AArch64::ZPRRegClass, Op0, Op0IsKill, imm1);
12187 }
12188 return 0;
12189}
12190
12191unsigned fastEmit_AArch64ISD_INDEX_VECTOR_ri_Predicate_simm5_8b(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
12192 switch (VT.SimpleTy) {
12193 case MVT::i32: return fastEmit_AArch64ISD_INDEX_VECTOR_MVT_i32_ri_Predicate_simm5_8b(RetVT, Op0, Op0IsKill, imm1);
12194 default: return 0;
12195 }
12196}
12197
12198// Top-level FastEmit function.
12199
12200unsigned fastEmit_ri_Predicate_simm5_8b(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
12201 switch (Opcode) {
12202 case AArch64ISD::INDEX_VECTOR: return fastEmit_AArch64ISD_INDEX_VECTOR_ri_Predicate_simm5_8b(VT, RetVT, Op0, Op0IsKill, imm1);
12203 default: return 0;
12204 }
12205}
12206
12207// FastEmit functions for AArch64ISD::INDEX_VECTOR.
12208
12209unsigned fastEmit_AArch64ISD_INDEX_VECTOR_MVT_i32_ri_Predicate_simm5_16b(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
12210 if (RetVT.SimpleTy != MVT::nxv8i16)
12211 return 0;
12212 if ((Subtarget->hasSVE())) {
12213 return fastEmitInst_ri(AArch64::INDEX_RI_H, &AArch64::ZPRRegClass, Op0, Op0IsKill, imm1);
12214 }
12215 return 0;
12216}
12217
12218unsigned fastEmit_AArch64ISD_INDEX_VECTOR_ri_Predicate_simm5_16b(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
12219 switch (VT.SimpleTy) {
12220 case MVT::i32: return fastEmit_AArch64ISD_INDEX_VECTOR_MVT_i32_ri_Predicate_simm5_16b(RetVT, Op0, Op0IsKill, imm1);
12221 default: return 0;
12222 }
12223}
12224
12225// Top-level FastEmit function.
12226
12227unsigned fastEmit_ri_Predicate_simm5_16b(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
12228 switch (Opcode) {
12229 case AArch64ISD::INDEX_VECTOR: return fastEmit_AArch64ISD_INDEX_VECTOR_ri_Predicate_simm5_16b(VT, RetVT, Op0, Op0IsKill, imm1);
12230 default: return 0;
12231 }
12232}
12233
12234// FastEmit functions for AArch64ISD::INDEX_VECTOR.
12235
12236unsigned fastEmit_AArch64ISD_INDEX_VECTOR_MVT_i32_ri_Predicate_simm5_32b(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
12237 if (RetVT.SimpleTy != MVT::nxv4i32)
12238 return 0;
12239 if ((Subtarget->hasSVE())) {
12240 return fastEmitInst_ri(AArch64::INDEX_RI_S, &AArch64::ZPRRegClass, Op0, Op0IsKill, imm1);
12241 }
12242 return 0;
12243}
12244
12245unsigned fastEmit_AArch64ISD_INDEX_VECTOR_ri_Predicate_simm5_32b(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
12246 switch (VT.SimpleTy) {
12247 case MVT::i32: return fastEmit_AArch64ISD_INDEX_VECTOR_MVT_i32_ri_Predicate_simm5_32b(RetVT, Op0, Op0IsKill, imm1);
12248 default: return 0;
12249 }
12250}
12251
12252// Top-level FastEmit function.
12253
12254unsigned fastEmit_ri_Predicate_simm5_32b(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
12255 switch (Opcode) {
12256 case AArch64ISD::INDEX_VECTOR: return fastEmit_AArch64ISD_INDEX_VECTOR_ri_Predicate_simm5_32b(VT, RetVT, Op0, Op0IsKill, imm1);
12257 default: return 0;
12258 }
12259}
12260
12261// FastEmit functions for AArch64ISD::INDEX_VECTOR.
12262
12263unsigned fastEmit_AArch64ISD_INDEX_VECTOR_MVT_i64_ri_Predicate_simm5_64b(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
12264 if (RetVT.SimpleTy != MVT::nxv2i64)
12265 return 0;
12266 if ((Subtarget->hasSVE())) {
12267 return fastEmitInst_ri(AArch64::INDEX_RI_D, &AArch64::ZPRRegClass, Op0, Op0IsKill, imm1);
12268 }
12269 return 0;
12270}
12271
12272unsigned fastEmit_AArch64ISD_INDEX_VECTOR_ri_Predicate_simm5_64b(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
12273 switch (VT.SimpleTy) {
12274 case MVT::i64: return fastEmit_AArch64ISD_INDEX_VECTOR_MVT_i64_ri_Predicate_simm5_64b(RetVT, Op0, Op0IsKill, imm1);
12275 default: return 0;
12276 }
12277}
12278
12279// Top-level FastEmit function.
12280
12281unsigned fastEmit_ri_Predicate_simm5_64b(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
12282 switch (Opcode) {
12283 case AArch64ISD::INDEX_VECTOR: return fastEmit_AArch64ISD_INDEX_VECTOR_ri_Predicate_simm5_64b(VT, RetVT, Op0, Op0IsKill, imm1);
12284 default: return 0;
12285 }
12286}
12287
12288// FastEmit functions for ISD::Constant.
12289
12290unsigned fastEmit_ISD_Constant_MVT_i32_i(MVT RetVT, uint64_t imm0) {
12291 if (RetVT.SimpleTy != MVT::i32)
12292 return 0;
12293 return fastEmitInst_i(AArch64::MOVi32imm, &AArch64::GPR32RegClass, imm0);
12294}
12295
12296unsigned fastEmit_ISD_Constant_MVT_i64_i(MVT RetVT, uint64_t imm0) {
12297 if (RetVT.SimpleTy != MVT::i64)
12298 return 0;
12299 return fastEmitInst_i(AArch64::MOVi64imm, &AArch64::GPR64RegClass, imm0);
12300}
12301
12302unsigned fastEmit_ISD_Constant_i(MVT VT, MVT RetVT, uint64_t imm0) {
12303 switch (VT.SimpleTy) {
12304 case MVT::i32: return fastEmit_ISD_Constant_MVT_i32_i(RetVT, imm0);
12305 case MVT::i64: return fastEmit_ISD_Constant_MVT_i64_i(RetVT, imm0);
12306 default: return 0;
12307 }
12308}
12309
12310// Top-level FastEmit function.
12311
12312unsigned fastEmit_i(MVT VT, MVT RetVT, unsigned Opcode, uint64_t imm0) override {
12313 if (VT == MVT::i32 && Predicate_imm0_255(imm0))
12314 if (unsigned Reg = fastEmit_i_Predicate_imm0_255(VT, RetVT, Opcode, imm0))
12315 return Reg;
12316
12317 switch (Opcode) {
12318 case ISD::Constant: return fastEmit_ISD_Constant_i(VT, RetVT, imm0);
12319 default: return 0;
12320 }
12321}
12322
12323// FastEmit functions for AArch64ISD::FMOV.
12324
12325unsigned fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v4f16_i_Predicate_imm0_255(uint64_t imm0) {
12326 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
12327 return fastEmitInst_i(AArch64::FMOVv4f16_ns, &AArch64::FPR64RegClass, imm0);
12328 }
12329 return 0;
12330}
12331
12332unsigned fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v8f16_i_Predicate_imm0_255(uint64_t imm0) {
12333 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
12334 return fastEmitInst_i(AArch64::FMOVv8f16_ns, &AArch64::FPR128RegClass, imm0);
12335 }
12336 return 0;
12337}
12338
12339unsigned fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v2f32_i_Predicate_imm0_255(uint64_t imm0) {
12340 if ((Subtarget->hasNEON())) {
12341 return fastEmitInst_i(AArch64::FMOVv2f32_ns, &AArch64::FPR64RegClass, imm0);
12342 }
12343 return 0;
12344}
12345
12346unsigned fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v4f32_i_Predicate_imm0_255(uint64_t imm0) {
12347 if ((Subtarget->hasNEON())) {
12348 return fastEmitInst_i(AArch64::FMOVv4f32_ns, &AArch64::FPR128RegClass, imm0);
12349 }
12350 return 0;
12351}
12352
12353unsigned fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v2f64_i_Predicate_imm0_255(uint64_t imm0) {
12354 if ((Subtarget->hasNEON())) {
12355 return fastEmitInst_i(AArch64::FMOVv2f64_ns, &AArch64::FPR128RegClass, imm0);
12356 }
12357 return 0;
12358}
12359
12360unsigned fastEmit_AArch64ISD_FMOV_MVT_i32_i_Predicate_imm0_255(MVT RetVT, uint64_t imm0) {
12361switch (RetVT.SimpleTy) {
12362 case MVT::v4f16: return fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v4f16_i_Predicate_imm0_255(imm0);
12363 case MVT::v8f16: return fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v8f16_i_Predicate_imm0_255(imm0);
12364 case MVT::v2f32: return fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v2f32_i_Predicate_imm0_255(imm0);
12365 case MVT::v4f32: return fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v4f32_i_Predicate_imm0_255(imm0);
12366 case MVT::v2f64: return fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v2f64_i_Predicate_imm0_255(imm0);
12367 default: return 0;
12368}
12369}
12370
12371unsigned fastEmit_AArch64ISD_FMOV_i_Predicate_imm0_255(MVT VT, MVT RetVT, uint64_t imm0) {
12372 switch (VT.SimpleTy) {
12373 case MVT::i32: return fastEmit_AArch64ISD_FMOV_MVT_i32_i_Predicate_imm0_255(RetVT, imm0);
12374 default: return 0;
12375 }
12376}
12377
12378// FastEmit functions for AArch64ISD::MOVI.
12379
12380unsigned fastEmit_AArch64ISD_MOVI_MVT_i32_MVT_v8i8_i_Predicate_imm0_255(uint64_t imm0) {
12381 if ((Subtarget->hasNEON())) {
12382 return fastEmitInst_i(AArch64::MOVIv8b_ns, &AArch64::FPR64RegClass, imm0);
12383 }
12384 return 0;
12385}
12386
12387unsigned fastEmit_AArch64ISD_MOVI_MVT_i32_MVT_v16i8_i_Predicate_imm0_255(uint64_t imm0) {
12388 if ((Subtarget->hasNEON())) {
12389 return fastEmitInst_i(AArch64::MOVIv16b_ns, &AArch64::FPR128RegClass, imm0);
12390 }
12391 return 0;
12392}
12393
12394unsigned fastEmit_AArch64ISD_MOVI_MVT_i32_i_Predicate_imm0_255(MVT RetVT, uint64_t imm0) {
12395switch (RetVT.SimpleTy) {
12396 case MVT::v8i8: return fastEmit_AArch64ISD_MOVI_MVT_i32_MVT_v8i8_i_Predicate_imm0_255(imm0);
12397 case MVT::v16i8: return fastEmit_AArch64ISD_MOVI_MVT_i32_MVT_v16i8_i_Predicate_imm0_255(imm0);
12398 default: return 0;
12399}
12400}
12401
12402unsigned fastEmit_AArch64ISD_MOVI_i_Predicate_imm0_255(MVT VT, MVT RetVT, uint64_t imm0) {
12403 switch (VT.SimpleTy) {
12404 case MVT::i32: return fastEmit_AArch64ISD_MOVI_MVT_i32_i_Predicate_imm0_255(RetVT, imm0);
12405 default: return 0;
12406 }
12407}
12408
12409// FastEmit functions for AArch64ISD::MOVIedit.
12410
12411unsigned fastEmit_AArch64ISD_MOVIedit_MVT_i32_MVT_f64_i_Predicate_imm0_255(uint64_t imm0) {
12412 return fastEmitInst_i(AArch64::MOVID, &AArch64::FPR64RegClass, imm0);
12413}
12414
12415unsigned fastEmit_AArch64ISD_MOVIedit_MVT_i32_MVT_v2i64_i_Predicate_imm0_255(uint64_t imm0) {
12416 if ((Subtarget->hasNEON())) {
12417 return fastEmitInst_i(AArch64::MOVIv2d_ns, &AArch64::FPR128RegClass, imm0);
12418 }
12419 return 0;
12420}
12421
12422unsigned fastEmit_AArch64ISD_MOVIedit_MVT_i32_i_Predicate_imm0_255(MVT RetVT, uint64_t imm0) {
12423switch (RetVT.SimpleTy) {
12424 case MVT::f64: return fastEmit_AArch64ISD_MOVIedit_MVT_i32_MVT_f64_i_Predicate_imm0_255(imm0);
12425 case MVT::v2i64: return fastEmit_AArch64ISD_MOVIedit_MVT_i32_MVT_v2i64_i_Predicate_imm0_255(imm0);
12426 default: return 0;
12427}
12428}
12429
12430unsigned fastEmit_AArch64ISD_MOVIedit_i_Predicate_imm0_255(MVT VT, MVT RetVT, uint64_t imm0) {
12431 switch (VT.SimpleTy) {
12432 case MVT::i32: return fastEmit_AArch64ISD_MOVIedit_MVT_i32_i_Predicate_imm0_255(RetVT, imm0);
12433 default: return 0;
12434 }
12435}
12436
12437// Top-level FastEmit function.
12438
12439unsigned fastEmit_i_Predicate_imm0_255(MVT VT, MVT RetVT, unsigned Opcode, uint64_t imm0) {
12440 switch (Opcode) {
12441 case AArch64ISD::FMOV: return fastEmit_AArch64ISD_FMOV_i_Predicate_imm0_255(VT, RetVT, imm0);
12442 case AArch64ISD::MOVI: return fastEmit_AArch64ISD_MOVI_i_Predicate_imm0_255(VT, RetVT, imm0);
12443 case AArch64ISD::MOVIedit: return fastEmit_AArch64ISD_MOVIedit_i_Predicate_imm0_255(VT, RetVT, imm0);
12444 default: return 0;
12445 }
12446}
12447
12448